package com.sino.dataworks.util;

import cn.hutool.core.date.BetweenFormatter;
import cn.hutool.core.date.DateField;
import cn.hutool.core.date.DateUnit;
import cn.hutool.core.date.DateUtil;
import cn.hutool.extra.ssh.JschUtil;
import cn.hutool.extra.ssh.Sftp;
import cn.hutool.http.Header;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.aliyun.odps.*;
import com.aliyun.odps.account.Account;
import com.aliyun.odps.account.AliyunAccount;
import com.aliyun.odps.data.ArrayRecord;
import com.aliyun.odps.data.Record;
import com.aliyun.odps.data.RecordReader;
import com.aliyun.odps.data.RecordWriter;
import com.aliyun.odps.task.SQLTask;
import com.aliyun.odps.tunnel.TableTunnel;
import com.aliyun.odps.tunnel.TunnelException;
import com.aliyuncs.CommonRequest;
import com.aliyuncs.DefaultAcsClient;
import com.aliyuncs.IAcsClient;
import com.aliyuncs.dataworks_public.model.v20200518.*;
import com.aliyuncs.exceptions.ClientException;
import com.aliyuncs.profile.DefaultProfile;
import com.aliyuncs.utils.StringUtils;
import com.google.gson.Gson;
import com.jcraft.jsch.SftpATTRS;
import com.sino.dataworks.cache.PhoneCache;
import com.sino.dataworks.dto.*;
import com.sino.dataworks.entity.DwDutyRoster;
import com.sino.dataworks.enums.DataWorksTaskInfoStatus;
import com.sino.dataworks.enums.DataWorksTaskStatus;
import com.sino.dataworks.enums.ProjectEnums;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.openqa.selenium.By;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.support.ui.ExpectedConditions;
import org.openqa.selenium.support.ui.WebDriverWait;

import java.io.*;
import java.math.BigDecimal;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.sql.Timestamp;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static com.sino.dataworks.util.JobVarUtil.*;
import static com.sino.dataworks.util.MultiSheetExcelExporter.createFieldMap;

@Slf4j
public class DataworksUtil {
    private static int retry=0;
    CommonRequest request = new CommonRequest();
    static String odpsUrl = "http://service.cn-wuhu-qrhy-d01.odps.ops.cloud.cheryfs.cn/api";
    static Account account= new AliyunAccount("2Z8355ERGTYosOtg", "4RYOuZWXDMtZ3VxI1euyGB68XmvNXD");
    static Account adminAccount= new AliyunAccount("rFloKx7rnzOYYaek", "rAT0Q9ZD7QKewCAILHIPn92J5a5m1M");
    static IAcsClient client = null;
    static IAcsClient slbClient=null;
    public static Map<String,String> type=new HashMap<>();
    static {
        String endPoint = System.getenv("ALIBABA_END_POINT");
        String regionId = System.getenv("ALIBABA_REGION_ID");
        if (StringUtils.isEmpty(endPoint)) {
            endPoint = "dataworks-public-vpc.res.cloud.cheryfs.cn";
        }
        String keyId = System.getenv("ALIBABA_CLOUD_ACCESS_KEY_ID");
        String keySec = System.getenv("ALIBABA_CLOUD_ACCESS_KEY_SECRET");
        if (StringUtils.isEmpty(keyId)) {
            keyId = "ZR18OFackif3jKT7";
        }
        if (StringUtils.isEmpty(keySec)) {
            keySec = "Pfvaw2BOmDPC4zEu1UglDu87jCn5yi";
        }
        if (StringUtils.isEmpty(regionId)) {
            regionId = "cn-wuhu-qrhy-d01";
        }
        DefaultProfile profile = DefaultProfile.getProfile(
                regionId,
                keyId, // 替换为你的AccessKey ID
                keySec); // 替换为你的AccessKey Secret
        DefaultProfile.addEndpoint(regionId, "dataworks-public", endPoint);
        client = new DefaultAcsClient(profile);
        String slbEndPoint="public.asapi.cn-wuhu-qrhy-d01.res.cloud.cheryfs.cn/asapi/v3";
        DefaultProfile slbProfile = DefaultProfile.getProfile(
                regionId,
                "feupFUuHYOxrohjn", // 替换为你的AccessKey ID
                "MUFrk9HsB94LXzyDZZA0lhPvvoa1J0"); // 替换为你的AccessKey Secret
        DefaultProfile.addEndpoint(regionId, "Slb", slbEndPoint);
        slbClient=new DefaultAcsClient(slbProfile);
        type.put("VARCHAR2","STRING");
        type.put("VARCHAR","STRING");
        type.put("NVARCHAR2","STRING");
        type.put("BIGINT","STRING");
        type.put("INTEGER","BIGINT");
        type.put("INT","BIGINT");
        type.put("CHAR","STRING");
        type.put("NUMBER","STRING");
        type.put("DECIMAL","STRING");
        type.put("RAW","BINARY");
        type.put("NCLOB","STRING");
        type.put("NCHAR","STRING");
        type.put("BLOB","STRING");
        type.put("CLOB","STRING");
        type.put("TIMESTAMP","TIMESTAMP");
        type.put("DATE","TIMESTAMP");
        type.put("DATETIME","TIMESTAMP");
        type.put("FLOAT","STRING");
        type.put("BIGSERIAL","STRING");
        type.put("INT4","STRING");
        type.put("INT8","STRING");
        type.put("FLOAT8","STRING");
    }
    public static String login(){
                // 设置ChromeDriver路径（如果已配置PATH则无需此步）
                StringBuffer stringBuffer=new StringBuffer();
                boolean isLogin=false;
                retry+=1;
                String kcookies="";
                System.setProperty("webdriver.chrome.driver", "chromedriver.exe");
                ChromeOptions options = new ChromeOptions();
                options.addArguments("--ignore-certificate-errors");
                WebDriver driver = new ChromeDriver(options);
                try {
                    String user=config.getStr("dw_user","Project_admin");
                    String password=config.getStr("dw_passwd","Chend123!@##");
                    // 打开DataWorks登录页面
                    driver.get("https://one.console.res.cloud.cheryfs.cn/ascm/login?oauth_callback=https%3A%2F%2Fcloudsso.res.cloud.cheryfs.cn%2Faliyun%2FaliyunCallback%3Fredirect%3Dhttp%253A%252F%252Fide.res.cloud.cheryfs.cn%252F");
                    Thread.sleep(5000);
                    // 填写用户名和密码
                    WebElement usernameInput = driver.findElement(By.id("account"));
                    WebElement passwordInput = driver.findElement(By.id("password"));

                    usernameInput.sendKeys(user); // 替换为你的用户名
                    passwordInput.sendKeys(password); // 替换为你的密码

                    // 提交表单
                    WebElement loginButton = driver.findElement(By.cssSelector(".next-form-item-control button[type='submit']"));
                    loginButton.click();
                    // 登录后可能需要进一步处理，如验证是否登录成功等
                    Thread.sleep(3000); // 同样，这里应该使用显式等待
                    try {
                        WebElement webElement=driver.findElement(By.cssSelector("body > div.next-overlay-wrapper.opened > div.next-dialog.next-closeable.next-overlay-inner.next-dialog-quick > div.next-dialog-footer.next-align-right > button.next-btn.next-medium.next-btn-normal.next-dialog-btn"));
                        webElement.click();
                        log.info("密码即将已到期");
                    }catch (Exception e){
                        log.info("密码未到期");
                    }
                    log.info(DateUtil.format(new Date(),"yyyy-MM-dd HH:mm:ss") +" 登录成功!");
                    Set<String> winHandels = driver.getWindowHandles();
                    log.info("窗口数量："+winHandels.size());
                    List<String> it =new ArrayList(winHandels);
                    log.info("窗口集合："+it.get(0));
                    driver.switchTo().window(it.get(0));
                    Thread.sleep(5000);
                    log.info("当前窗口："+driver.getCurrentUrl());
                    WebDriverWait webDriverWait=new WebDriverWait(driver, 50);
                    WebElement webElement=driver.findElement(By.xpath("//*[@id=\"icestarkNode\"]/section/div/div[2]/div[3]/span/button"));
                    log.info("按钮是否可以点击:"+webElement.isEnabled());
                    if(!webElement.isEnabled()){
                        WebElement button =webDriverWait.until(ExpectedConditions.elementToBeClickable(By.xpath("/html/body/div[1]/div[2]/main/div/div/div/section/div/div[2]/div[1]/div/span/span[1]/span[1]")));
                        button.click();
                        WebElement buttons=webDriverWait.until(ExpectedConditions.elementToBeClickable(By.xpath("/html/body/div[4]/div/div/div/div/div/ul/li[3]/div/div/div")));
                        buttons.click();
                    }
                    WebElement button =webDriverWait.until(ExpectedConditions.elementToBeClickable(By.xpath("//*[@id=\"icestarkNode\"]/section/div/div[2]/div[3]/span/button")));
                    button.click();
                    driver.get("https://ide.res.cloud.cheryfs.cn");
                    Thread.sleep(5000);
                    Set<Cookie> cookies = driver.manage().getCookies();
                    for (Cookie cookie : cookies) {
                        log.info(cookie.getName() + "=" + cookie.getValue());
                        stringBuffer.append(cookie.getName() + "=" + cookie.getValue()+"; ");
                        isLogin=true;
                    }
                    if(isLogin){
                        retry=0;
                    }
                     kcookies=stringBuffer.toString().trim();
                     kcookies=kcookies.substring(0, kcookies.lastIndexOf(";"));
                    Path path = Paths.get("cookie.txt");
                    if (!Files.exists(path)) {
                        Files.createFile(path);
                    }
                    Files.write(path, kcookies.getBytes(), StandardOpenOption.TRUNCATE_EXISTING);
                } catch (Exception e) {
                    e.printStackTrace();
                } finally {
                    log.info("第"+(retry+1)+"次登陆");
                    if(retry>=3){
                        log.info("尝试登陆"+(retry+1)+"次失败");
                        phoneWarning("登录数据中台失败");
                        throw new RuntimeException("登录失败");
                    }
                    // 关闭浏览器
                    driver.quit();
                }
                return kcookies;
     }


   public static List<InstanceEntityDto> getInstance(int status,String bizDate,String currentDate){
        List<InstanceEntityDto> instanceEntityDtos=new ArrayList<>();
       File files = new File("cookie.txt");
       try {
           String cookies=readerMethod(files);
           String url="https://wkbench.res.cloud.cheryfs.cn/workbench/cwf/instance/list?projectId=10022&env=prod&tenantId=631873240849152&sortOrder=&sortField=&includeRelation=false&dagType=0&taskStatuses="+status+"&pageNum=1&pageSize=100&forceUpdateId=5&expand=true&prgTypes=&searchText=&owner=&solId=&bizId=&taskTypes=0&startRunTimeFrom=&startRunTimeTo=&baseLineId=&bizdate="+bizDate+"%2000%3A00%3A00&beginBizDate=&endBizDate=&bizBeginHour="+currentDate+"%2000%3A00%3A00&bizEndHour="+currentDate+"%2023%3A59%3A00&connectionRegionId=&connectionType=&connections=&slowly=false&diResGroupIdentifier=&diSrcType=&diDstType=&diSrcDatasource=&diDstDatasource=&resgroupId=";
           String result3= HttpRequest.get(url)
                   .cookie(cookies)
                   .execute().body();
          log.info("查询状态:"+status+"  查询结果:"+result3);
          if(StringUtils.isNotEmpty(result3)){
              JSONObject jsonObject= JSON.parseObject(result3);
              if(jsonObject.getBoolean("success")){
                  JSONObject data=jsonObject.getJSONObject("data");
                  JSONArray list=data.getJSONArray("data");
                   for (int i = 0; i < list.size(); i++){
                       JSONObject jsonObject1=list.getJSONObject(i);
                       Gson gson = new Gson();
                       InstanceEntityDto instanceEntityDto = gson.fromJson(JSON.toJSONString(jsonObject1), InstanceEntityDto.class);
                       if(jsonObject1.getLong("taskId")!=null){
                           instanceEntityDto.setTaskId(jsonObject1.getLong("taskId"));
                       }
                       instanceEntityDtos.add(instanceEntityDto);
                   }
              }
          }else{
              log.info("查询结果为空 尝试重新登陆");
              login();
              return getInstance(status,bizDate,currentDate);
          }
       } catch (IOException e) {
           throw new RuntimeException(e);
       }
       log.info("获取实例结果:"+JSON.toJSONString(instanceEntityDtos));
       return instanceEntityDtos;
   }


    public static List<InstanceEntityDto> getErrorInstance(int status,String bizDate,String currentDate,int page,int pageSize){
        List<InstanceEntityDto> instanceEntityDtos=new ArrayList<>();
        File files = new File("cookie.txt");
        try {
            String cookies=readerMethod(files);
            String url="https://wkbench.res.cloud.cheryfs.cn/workbench/cwf/instance/list?projectId=10022&env=prod&tenantId=631873240849152&sortOrder=&sortField=&includeRelation=false&dagType=0&taskStatuses="+status+"&pageNum="+page+"&pageSize=100&forceUpdateId=5&expand=true&prgTypes=&searchText=&owner=&solId=&bizId=&taskTypes=0&startRunTimeFrom=&startRunTimeTo=&baseLineId=&bizdate="+bizDate+"%2000%3A00%3A00&beginBizDate=&endBizDate=&bizBeginHour="+currentDate+"%2000%3A00%3A00&bizEndHour="+currentDate+"%2023%3A59%3A00&connectionRegionId=&connectionType=&connections=&slowly=false&diResGroupIdentifier=&diSrcType=&diDstType=&diSrcDatasource=&diDstDatasource=&resgroupId=";
            String result3= HttpRequest.get(url)
                    .cookie(cookies)
                    .execute().body();
            log.info("查询状态:"+status+"  查询结果:"+result3);
            if(StringUtils.isNotEmpty(result3)){
                JSONObject jsonObject= JSON.parseObject(result3);
                if(jsonObject.getBoolean("success")){
                    JSONObject data=jsonObject.getJSONObject("data");
                    JSONArray list=data.getJSONArray("data");
                    for (int i = 0; i < list.size(); i++){
                        JSONObject jsonObject1=list.getJSONObject(i);
                        Gson gson = new Gson();
                        InstanceEntityDto instanceEntityDto = gson.fromJson(JSON.toJSONString(jsonObject1), InstanceEntityDto.class);
                        if(jsonObject1.getLong("taskId")!=null){
                            instanceEntityDto.setTaskId(jsonObject1.getLong("taskId"));
                        }else{
                            log.info("获取taskId为空："+jsonObject1.getString("nodeName"));
                        }
                        instanceEntityDtos.add(instanceEntityDto);
                    }
                }
            }else{
                log.info("查询结果为空 尝试重新登陆");
                login();
                return getInstance(status,bizDate,currentDate);
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        log.info("获取实例结果:"+JSON.toJSONString(instanceEntityDtos));
        return instanceEntityDtos;
    }

    public static List<InstanceEntityDto> getInstance(String nodeName,String bizDate,String currentDate){
        List<InstanceEntityDto> instanceEntityDtos=new ArrayList<>();
        File files = new File("cookie.txt");
        try {
            String cookies=readerMethod(files);
            String url="https://wkbench.res.cloud.cheryfs.cn/workbench/cwf/instance/list?projectId=10022&env=prod&tenantId=631873240849152&sortOrder=&sortField=&includeRelation=false&dagType=0&searchText="+nodeName+"&pageNum=1&pageSize=100&forceUpdateId=5&expand=true&prgTypes=&taskStatuses=&owner=&solId=&bizId=&taskTypes=0&startRunTimeFrom=&startRunTimeTo=&baseLineId=&bizdate="+bizDate+"%2000%3A00%3A00&beginBizDate=&endBizDate=&bizBeginHour="+currentDate+"%2000%3A00%3A00&bizEndHour="+currentDate+"%2023%3A59%3A00&connectionRegionId=&connectionType=&connections=&slowly=false&diResGroupIdentifier=&diSrcType=&diDstType=&diSrcDatasource=&diDstDatasource=&resgroupId=";
            String result3= HttpRequest.get(url)
                    .cookie(cookies)
                    .execute().body();
            if(StringUtils.isNotEmpty(result3)){
                JSONObject jsonObject= JSON.parseObject(result3);
                if(jsonObject.getBoolean("success")){
                    JSONObject data=jsonObject.getJSONObject("data");
                    JSONArray list=data.getJSONArray("data");
                    for (int i = 0; i < list.size(); i++){
                        JSONObject jsonObject1=list.getJSONObject(i);
                        Gson gson = new Gson();
                        InstanceEntityDto instanceEntityDto = gson.fromJson(JSON.toJSONString(jsonObject1), InstanceEntityDto.class);
                        if(jsonObject1.getLong("taskId")!=null){
                            instanceEntityDto.setTaskId(jsonObject1.getLong("taskId"));
                        }else{
                            log.info("获取taskId为空："+jsonObject1.getString("nodeName"));
                        }
                        instanceEntityDtos.add(instanceEntityDto);
                    }
                }
            }else{
                log.info("查询结果为空 尝试重新登陆");
                login();
                return getInstance(nodeName,bizDate,currentDate);
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        log.info("获取实例结果:"+JSON.toJSONString(instanceEntityDtos));
        return instanceEntityDtos;
    }

    public static List<InstanceEntityDto> getInstance(String searchText,String bizDate,String currentDate,int page,int pageSize){
        List<InstanceEntityDto> instanceEntityDtos=new ArrayList<>();
        File files = new File("cookie.txt");
        try {
            String cookies=readerMethod(files);
            String url="https://wkbench.res.cloud.cheryfs.cn/workbench/cwf/instance/list?projectId=10022&env=prod&tenantId=631873240849152&sortOrder=&sortField=&includeRelation=false&dagType=0&searchText="+searchText+"&pageNum="+page+"&pageSize="+pageSize+"&pageSize=100&forceUpdateId=5&expand=true&prgTypes=&taskStatuses=&owner=&solId=&bizId=&taskTypes=0&startRunTimeFrom=&startRunTimeTo=&baseLineId=&bizdate="+bizDate+"%2000%3A00%3A00&beginBizDate=&endBizDate=&bizBeginHour="+currentDate+"%2000%3A00%3A00&bizEndHour="+currentDate+"%2023%3A59%3A00&connectionRegionId=&connectionType=&connections=&slowly=false&diResGroupIdentifier=&diSrcType=&diDstType=&diSrcDatasource=&diDstDatasource=&resgroupId=";
            String result3= HttpRequest.get(url)
                    .cookie(cookies)
                    .execute().body();
            if(StringUtils.isNotEmpty(result3)){
                JSONObject jsonObject= JSON.parseObject(result3);
                if(jsonObject.getBoolean("success")){
                    JSONObject data=jsonObject.getJSONObject("data");
                    JSONArray list=data.getJSONArray("data");
                    for (int i = 0; i < list.size(); i++){
                        JSONObject jsonObject1=list.getJSONObject(i);
                        Gson gson = new Gson();
                        InstanceEntityDto instanceEntityDto = gson.fromJson(JSON.toJSONString(jsonObject1), InstanceEntityDto.class);
                        if(jsonObject1.getLong("taskId")!=null){
                            instanceEntityDto.setTaskId(jsonObject1.getLong("taskId"));
                        }else{
                            log.info("获取taskId为空："+jsonObject1.getString("nodeName"));
                        }
                        instanceEntityDtos.add(instanceEntityDto);
                    }
                }
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        log.info("获取实例结果:"+JSON.toJSONString(instanceEntityDtos));
        return instanceEntityDtos;
    }

    public static List<InstanceEntityDto> getNodeList(Map<String,String> param){
        List<InstanceEntityDto> instanceEntityDtos=new ArrayList<>();
        File files = new File("cookie.txt");
        Map<String, String> params = new HashMap<>();
        try {
            String cookies=readerMethod(files);
            params.put("projectId", "10022");
            params.put("env", "prod");
            params.put("tenantId", "631873240849152");
            params.put("sortOrder", "");
            params.put("sortField", "");
            params.put("includeRelation", "false");
            params.put("pageNum", "1");
            params.put("pageSize", "100");
            params.put("forceUpdateId", "2");
            params.put("expand", "true");
            params.put("prgTypes", "");
            params.put("modifyTime", "");
            params.put("nodeType", "");
            params.put("owner", "");
            params.put("solId", "");
            params.put("resgroupId", "");
            params.put("baseLineId", "");
            params.put("connectionRegionId", "");
            params.put("connectionType", "");
            params.put("connections", "");
            params.put("expired", "false");
            params.put("lonely", "false");
            params.put("diResGroupIdentifier", "");
            params.put("diSrcType", "");
            params.put("diDstType", "");
            params.put("diSrcDatasource", "");
            params.put("diDstDatasource", "");
            params.putAll(param);
            String baseUrl="https://wkbench.res.cloud.cheryfs.cn/workbench/cwf/node/list";
            String url = buildUrlWithParams(baseUrl, params);
            log.info("参数列表:"+JSON.toJSONString(params));
//            if(StringUtils.isNotEmpty(searchText)){
//                url+="&searchText="+searchText;
//            }else {
//                url+="&searchText=";
//            }
//            if(StringUtils.isNotEmpty(bizId)){
//                url+="&bizId="+bizId;
//            } else {
//                url+="&bizId=";
//             }
            String result3= HttpRequest.get(url)
                    .cookie(cookies)
                    .execute().body();
            if(StringUtils.isNotEmpty(result3)){
                JSONObject jsonObject= JSON.parseObject(result3);
                if(jsonObject.getBoolean("success")){
                    JSONObject data=jsonObject.getJSONObject("data");
                    JSONArray list=data.getJSONArray("data");
                    JobVarUtil.nodeListTotalCount=data.getInteger("totalNum");
                    for (int i = 0; i < list.size(); i++){
                        JSONObject jsonObject1=list.getJSONObject(i);
                        Gson gson = new Gson();
                        InstanceEntityDto instanceEntityDto = gson.fromJson(JSON.toJSONString(jsonObject1), InstanceEntityDto.class);
                        if(jsonObject1.getLong("taskId")!=null){
                            instanceEntityDto.setTaskId(jsonObject1.getLong("taskId"));
                        }else{
                            log.info("获取taskId为空："+jsonObject1.getString("nodeName"));
                        }
                        instanceEntityDtos.add(instanceEntityDto);
                    }
                }
            }else{
                login();
                getNodeList(params);
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return instanceEntityDtos;
    }

    private static String readerMethod(File file) throws IOException {
        FileReader fileReader = new FileReader(file);
        InputStreamReader reader = new InputStreamReader(new FileInputStream(file), "Utf-8");
        int ch= 0;
        StringBuffer sb = new StringBuffer();
        while((ch = reader.read()) != -1) {
            sb.append((char) ch);
        }
        fileReader.close();
        reader.close();
        return  sb.toString();
    }
    public  static List<ListInstancesResponse.Data.Instance> getInstancesByNodeId(String nodeId) {
        // 创建DefaultAcsClient实例并初始化
        List<ListInstancesResponse.Data.Instance> instances=new ArrayList<>();
        StringBuffer stringBuffer = new StringBuffer();
        Date newDate = DateUtil.offset(new Date(), DateField.DAY_OF_MONTH, -1);
        String startTime = DateUtil.format(newDate, "yyyy-MM-dd") + " 00:00:00";
        String endTime = DateUtil.format(newDate, "yyyy-MM-dd") + " 00:00:00";
        String time = DateUtil.format(newDate, "yyyyMMdd");
        // 创建API请求并设置参数
        ListInstancesRequest request = new ListInstancesRequest();
        request.setProjectId(10022L); // 替换为你的DataWorks项目名称
        request.setProjectEnv("PROD");
        request.setBeginBizdate(startTime);
        request.setEndBizdate(endTime);
        request.setPageSize(100);
        request.setNodeId(Long.valueOf(nodeId));
        request.setPageNumber(1);
        // 可以根据需要设置更多参数，如pageNumber、pageSize等
        try {
            // 调用ListInstances方法
            ListInstancesResponse response = client.getAcsResponse(request);
            // 处理响应结果
            log.info("根据NodeId查询实例结果:"+JSON.toJSONString(response));
            if (response != null && response.getSuccess()) {
                instances = response.getData().getInstances();
            } else {
                log.info("Failed to list instances. Error code: " + response.getErrorCode() + ", Error message: " + response.getErrorMessage());
            }
        } catch (ClientException e) {
            e.printStackTrace();
        }
        return instances;
    }

    public  static List<Record> getRecordByNodeId(String nodeId) {
        // 创建DefaultAcsClient实例并初始化
        long timestamp = 1627096274000L;
        Date date = new Date(timestamp);
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        List<Record> instances=new ArrayList<>();
        StringBuffer stringBuffer = new StringBuffer();
        Date newDate = DateUtil.offset(new Date(), DateField.DAY_OF_MONTH, -1);
        String startTime = DateUtil.format(newDate, "yyyy-MM-dd") + " 00:00:00";
        String endTime = DateUtil.format(newDate, "yyyy-MM-dd") + " 00:00:00";
        String time = DateUtil.format(newDate, "yyyyMMdd");
        // 创建API请求并设置参数
        ListInstancesRequest request = new ListInstancesRequest();
        request.setProjectId(10022L); // 替换为你的DataWorks项目名称
        request.setProjectEnv("PROD");
        request.setBeginBizdate(startTime);
        request.setEndBizdate(endTime);
        request.setPageSize(100);
        request.setNodeId(Long.valueOf(nodeId));
        request.setPageNumber(1);
        // 可以根据需要设置更多参数，如pageNumber、pageSize等
        try {
            // 调用ListInstances方法
            ListInstancesResponse response = client.getAcsResponse(request);
            // 处理响应结果
            log.info("根据NodeId查询实例结果:"+JSON.toJSONString(response));
            if (response != null && response.getSuccess()) {
              List<ListInstancesResponse.Data.Instance>  instanceList = response.getData().getInstances();
              for (ListInstancesResponse.Data.Instance instance:instanceList){
                  Column[] columns1=new Column[27];
                  columns1[0]=(new Column("nodeId", OdpsType.valueOf("BIGINT")));
                  columns1[1]=(new Column("instanceId", OdpsType.valueOf("BIGINT")));
                  columns1[2]=(new Column("dagId", OdpsType.valueOf("BIGINT")));
                  columns1[3]=(new Column("dagType", OdpsType.valueOf("STRING")));
                  columns1[4]=(new Column("status", OdpsType.valueOf("STRING")));
                  columns1[5]=(new Column("bizdate", OdpsType.valueOf("STRING")));
                  columns1[6]=(new Column("cycTime", OdpsType.valueOf("TIMESTAMP")));
                  columns1[7]=(new Column("createTime", OdpsType.valueOf("TIMESTAMP")));
                  columns1[8]=(new Column("modifyTime", OdpsType.valueOf("TIMESTAMP")));
                  columns1[9]=(new Column("nodeName", OdpsType.valueOf("STRING")));
                  columns1[10]=(new Column("beginWaitTimeTime", OdpsType.valueOf("TIMESTAMP")));
                  columns1[11]=(new Column("beginWaitResTime", OdpsType.valueOf("TIMESTAMP")));
                  columns1[12]=(new Column("beginRunningTime", OdpsType.valueOf("TIMESTAMP")));
                  columns1[13]=(new Column("paramValues", OdpsType.valueOf("STRING")));
                  columns1[14]=(new Column("finishTime", OdpsType.valueOf("TIMESTAMP")));
                  columns1[15]=(new Column("priority", OdpsType.valueOf("BIGINT")));
                  columns1[16]=(new Column("baselineId", OdpsType.valueOf("BIGINT")));
                  columns1[17]=(new Column("repeatability", OdpsType.valueOf("BOOLEAN")));
                  columns1[18]=(new Column("repeatInterval", OdpsType.valueOf("BIGINT")));
                  columns1[19]=(new Column("connection", OdpsType.valueOf("STRING")));
                  columns1[20]=(new Column("dqcType", OdpsType.valueOf("BIGINT")));
                  columns1[21]=(new Column("dqcDescription", OdpsType.valueOf("STRING")));
                  columns1[22]=(new Column("errorMessage", OdpsType.valueOf("STRING")));
                  columns1[23]=(new Column("relatedFlowId", OdpsType.valueOf("BIGINT")));
                  columns1[24]=(new Column("taskType", OdpsType.valueOf("STRING")));
                  columns1[25]=(new Column("taskRerunTime", OdpsType.valueOf("BIGINT")));
                  columns1[26]=(new Column("businessId", OdpsType.valueOf("BIGINT")));
                  Record record=new ArrayRecord(columns1);
                  record.setBigint(0,instance.getNodeId());
                  record.setBigint(1,instance.getInstanceId());
                  record.setBigint(2,instance.getDagId());
                  record.setString(3,instance.getDagType());
                  record.setString(4,instance.getStatus());
                  record.setString(5, sdf.format(new Date(instance.getBizdate())));
                  record.setDatetime(6,new Timestamp(instance.getCycTime()));
                  record.setDatetime(7,new Timestamp(instance.getCreateTime()));
                  record.setDatetime(8,new Timestamp(instance.getModifyTime()));
                  record.setString(9,instance.getNodeName());
                  record.setDatetime(10,new Timestamp(instance.getBeginWaitTimeTime()));
                  record.setDatetime(11,new Timestamp(instance.getBeginWaitResTime()));
                  record.setDatetime(12,new Timestamp(instance.getBeginRunningTime()));
                  record.setString(13,instance.getParamValues());
                  record.setDatetime(14,new Timestamp(instance.getFinishTime()));
                  record.setBigint(15,instance.getPriority().longValue());
                  record.setBigint( 16,instance.getBaselineId());
                  record.setBoolean(17,instance.getRepeatability());
                  record.setBigint(18,instance.getRepeatInterval());
                  record.setString(19,instance.getConnection());
                  if(instance.getDqcType()!=null){
                      record.setBigint(20,instance.getDqcType().longValue());
                  }else{
                      record.setBigint(20,0L);
                  }
                  record.setString(21,instance.getDqcDescription());
                  record.setString(22,instance.getErrorMessage());
                  record.setBigint(23,instance.getRelatedFlowId());
                  record.setString(24,instance.getTaskType());
                  if(instance.getTaskRerunTime()!=null){
                      record.setBigint(25,instance.getTaskRerunTime().longValue());
                  }else {
                      record.setBigint(25,0L);
                  }
                  record.setBigint(26,instance.getBaselineId());
                  instances.add(record);
              }
            } else {
                log.info("Failed to list instances. Error code: " + response.getErrorCode() + ", Error message: " + response.getErrorMessage());
            }
        } catch (ClientException e) {
            e.printStackTrace();
        }
        return instances;
    }
    public static Map<String,InstanceEntityDto> reRun(List<InstanceEntityDto> list){
        JSONObject app=new JSONObject();
        Map<String,InstanceEntityDto> mp=new HashMap<>();
        JSONArray taskIds=new JSONArray();
        File files = new File("cookie.txt");
        String cookies= null;
        try {
            cookies = readerMethod(files);
            Long projectId=10022L;
            String env="prod";
            app.put("projectId",projectId);
            app.put("env",env);
            Long tenantId=631873240849152L;
            app.put("tenantId",tenantId);
            for (InstanceEntityDto instanceEntityDto:list){
             //   if(instanceEntityDto.isRerunAble()){
                    log.info("重跑任务:"+instanceEntityDto.getNodeName());
                    mp.put(instanceEntityDto.getNodeName(),instanceEntityDto);
                String setSuccess= config.getStr("setSuccess");
                if(StringUtils.isNotEmpty(setSuccess)&&Arrays.asList(setSuccess).contains(instanceEntityDto.getNodeName())){
                    String errorLogs=DataworksUtil.getInstanceLog(instanceEntityDto.getTaskId());
                    String logDesc= LogParserUtil.getlogErrorInfo(errorLogs);
                    if(StringUtils.isNotEmpty(logDesc)&&logDesc.contains("overflows to convert to")){
                        Long instanceId=instanceEntityDto.getTaskId();
                        DataworksUtil.setSuccessInstance(instanceId);
                      }else{
                        taskIds.add(instanceEntityDto.getTaskId());
                    }
                   }else{
                    taskIds.add(instanceEntityDto.getTaskId());
                   }

             //   }
            }
            app.put("taskIds",taskIds);
            String token=getcsrfToken();
            HttpRequest request= HttpRequest.post("https://wkbench.res.cloud.cheryfs.cn/workbench/cwf/instance/rerun?projectId="+projectId+"&env="+env+"&tenantId="+tenantId);
           String result= request.header(Header.CONTENT_TYPE, "application/json")
                   .header(Header.ACCEPT, "application/json")
                   .header(Header.COOKIE, cookies+"; "+"csrf-token="+token)
                   .header("x-csrf-token",token)
                    .body(app.toJSONString())
                    .execute().body();
           if(StringUtils.isNotEmpty(result)){
               if(JSON.parseObject(result).getBoolean("success")){
                   log.info("重跑成功");
               }else{
                   log.info("重跑失败:"+JSON.parseObject(result).getString("errMsg"));
               }
           }
           log.info("重跑请求地址:"+request.getUrl()+"  "+request);
            log.info("重跑请求参数:"+app.toJSONString());
            log.info("重跑返回结果:"+result);
        } catch (IOException e) {
            log.info("重跑发生异常:"+e.getMessage());
            throw new RuntimeException(e);
        }
        return mp;
    }

    public static String getSyncTaskDetail(int page){
        File files = new File("cookie.txt");
        String res=null;
        try {
            Date newDate = DateUtil.offset(new Date(), DateField.DAY_OF_MONTH, 0);
            String startTime = DateUtil.format(newDate, "yyyy-MM-dd") + " 00:00:00";
            String endTime = DateUtil.format(newDate, "yyyy-MM-dd") + " 23:00:00";
            long head = DateUtil.parse(startTime).getTime();
            long end= DateUtil.parse(endTime).getTime();
            String cookies=readerMethod(files);
            String url="https://wkbench.res.cloud.cheryfs.cn/dwproxy/web/di/instance/list?projectId=10022&env=prod&tenantId=631873240849152&productCode=di&state=SUCCESS&submitTimeHead="+head+"&submitTimeEnd="+end+"&pageSize=100&pageNum="+page;
             String result3= HttpRequest.get(url)
                    .cookie(cookies)
                    .execute().body();
             res=result3;
            if(StringUtils.isNotEmpty(result3)){
                log.info("查询集成推送结果:"+result3);
                if(result3.contains("<html>")){
                    retryCount++;
                    try {
                        Thread.sleep(1000*60);
                        if(retryCount<=5){
                            log.info("第"+retryCount+"次尝试重新获取集成推送结果:"+result3);
                           return   getSyncTaskDetail(page);
                        }
                    } catch (InterruptedException e) {
                        throw new RuntimeException(e);
                    }
                }
                JSONObject jsonObject= JSON.parseObject(result3);
                if(jsonObject.getInteger("errCode")==0){
                    JSONObject data=jsonObject.getJSONObject("data");
                    instaceListTotalCount=data.getInteger("totalNum");
                    JSONArray list=data.getJSONArray("data");
                    for (int i = 0; i < list.size(); i++){
                        JSONObject jsonObject1=list.getJSONObject(i);
                        if(!instanceSyncTaskMap.containsKey(jsonObject1.getString("nodeName"))){
                            instanceSyncTaskMap.put(jsonObject1.getString("nodeName"),jsonObject1);
                        }
                    }
                }
            }else{
                log.info("查询结果为空 尝试重新登陆");
                login();
              return   getSyncTaskDetail(page);
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
         return res;
    }

    public static List<InstanceEntityDto> getInstance(String status,String bizDate,String currentDate,String name){
        List<InstanceEntityDto> instanceEntityDtos=new ArrayList<>();
        File files = new File("cookie.txt");
        try {
            String cookies=readerMethod(files);
            String url="https://wkbench.res.cloud.cheryfs.cn/workbench/cwf/instance/list?projectId=10022&env=prod&tenantId=631873240849152&sortOrder=&sortField=&includeRelation=false&dagType=0&taskStatuses="+status+"&pageNum=1&pageSize=100&forceUpdateId=5&expand=true&prgTypes=&searchText="+name+"&owner=&solId=&bizId=&taskTypes=0&startRunTimeFrom=&startRunTimeTo=&baseLineId=&bizdate="+bizDate+"%2000%3A00%3A00&beginBizDate=&endBizDate=&bizBeginHour="+currentDate+"%2000%3A00%3A00&bizEndHour="+currentDate+"%2023%3A59%3A00&connectionRegionId=&connectionType=&connections=&slowly=false&diResGroupIdentifier=&diSrcType=&diDstType=&diSrcDatasource=&diDstDatasource=&resgroupId=";
            String result3= HttpRequest.get(url)
                    .cookie(cookies)
                    .execute().body();
            log.info("查询状态:"+status+"  查询结果:"+result3);
            if(StringUtils.isNotEmpty(result3)){
                JSONObject jsonObject= JSON.parseObject(result3);
                if(jsonObject.getBoolean("success")){
                    JSONObject data=jsonObject.getJSONObject("data");
                    JSONArray list=data.getJSONArray("data");
                    for (int i = 0; i < list.size(); i++){
                        JSONObject jsonObject1=list.getJSONObject(i);
                        Gson gson = new Gson();
                        InstanceEntityDto instanceEntityDto = gson.fromJson(JSON.toJSONString(jsonObject1), InstanceEntityDto.class);
                        instanceEntityDtos.add(instanceEntityDto);
                    }
                }
            }else{
                log.info("查询结果为空 尝试重新登陆");
                login();
                return getInstance(status,bizDate,currentDate,name);
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        log.info("获取实例结果:"+JSON.toJSONString(instanceEntityDtos));
        return instanceEntityDtos;
    }
    private static String getcsrfToken(){
        String token=null;
        File files = new File("cookie.txt");
        try {
            String cookies=readerMethod(files);
            String url="https://wkbench.res.cloud.cheryfs.cn/csrf.token";
            String result= HttpRequest.get(url)
                    .cookie(cookies)
                    .execute().body();
            log.info("获取csrftoken:"+"  结果:"+result);
            if(StringUtils.isNotEmpty(result)) {
                token=result.trim().toString();
            }else {
                login();
                return getcsrfToken();
            }
            }catch (Exception e){
            e.printStackTrace();
        }
        return token;
      }

    private static String getxcsrfToken(){
        String token=null;
        File files = new File("cookie.txt");
        try {
            String cookies=readerMethod(files);
            String url="https://ide.res.cloud.cheryfs.cn/csrf.token";
            String result= HttpRequest.get(url)
                    .cookie(cookies)
                    .execute().body();
            log.info("获取csrftoken:"+"  结果:"+result);
            if(StringUtils.isNotEmpty(result)) {
                token=result.trim().toString();
            }else {
                login();
                return getcsrfToken();
            }
        }catch (Exception e){
            e.printStackTrace();
        }
        return token;
    }
    public static void phoneWarning(String content) {
        Date newDate = DateUtil.offset(new Date(), DateField.DAY_OF_MONTH, 0);
        String time = DateUtil.format(newDate, "yyyyMMdd");
        DwDutyRoster dwDutyRoster=new DwDutyRoster();
        try {
            if (!PhoneCache.phoneCache.containsKey(time)) {
                PhoneCache.initPhone();
            }
            dwDutyRoster = PhoneCache.phoneCache.get(time);
        }catch (Exception e){
            // e.printStackTrace();
            String phone=config.getStr("exceptWarnPhone","18838098890");
            dwDutyRoster.setFirstPhone(phone);
            dwDutyRoster.setSecondPhone(phone);
            dwDutyRoster.setThirdPhone(phone);
            dwDutyRoster.setFirstChargePerson("王晓东");
            dwDutyRoster.setSecondChargePerson("王晓东");
            dwDutyRoster.setThirdChargePerson("王晓东");
        }
        String phone1 = dwDutyRoster.getFirstPhone();
        String phone2 = dwDutyRoster.getSecondPhone();
        String phone3 = dwDutyRoster.getThirdPhone();
        String alarmContent = "数据中台跑批节点数据跑批失败，请及时处理";
        if(StringUtils.isNotEmpty(content)){
            alarmContent=content;
        }
        log.info(alarmContent);
        log.info("第一顺序" + dwDutyRoster.getFirstChargePerson() + ":告警电话:" + phone1 + " 第二顺序" + dwDutyRoster.getSecondChargePerson() + ":告警电话：" + phone2 + " 第三顺序" + dwDutyRoster.getThirdChargePerson() + ":告警电话:" + phone3);
        String url = "https://aicc.cheryfs.cn/api/v1/config/openapi/contact/batch/json/upload";
        String token = "qD9w5lF8i8Dtn62mvxIRpaGP5FyrZiunGcoQB2JHKFuzB6T4BGwT2MqptoYECkPg";
        String dataSourceName = "system_alarm";
        String dataListElement = "" + phone1 + ",637839444jg76e72378,637839444jg76e72378,数据中台,智能外呼,跑批节点," + alarmContent + "," + phone2 + "," + phone3 + "";
        String abbreviate = "cheryfs";
        Map<String, String> headers = new HashMap<>();
        headers.put("token", token);
        headers.put("Content-Type", "application/json");
        JSONObject data = new JSONObject();
        data.put("dataSourceName", dataSourceName);
        JSONArray one = new JSONArray();
        String[] two = dataListElement.split(",");
        one.add(two);
        data.put("dataList", one);
        data.put("abbreviate", abbreviate);
        log.info("告警参数：" + JSON.toJSONString(data));
        HttpResponse response = HttpRequest.post(url)
                .headerMap(headers, false)
                .body(JSON.toJSONString(data))
                .execute();
        log.info("告警结果返回:" + response);
    }

    public static void phoneWarning(String content,int intervalMinute) {
        Date newDate = DateUtil.offset(new Date(), DateField.DAY_OF_MONTH, 0);
        String time = DateUtil.format(newDate, "yyyyMMdd");
        DwDutyRoster dwDutyRoster=new DwDutyRoster();
        try {
            if (!PhoneCache.phoneCache.containsKey(time)) {
                PhoneCache.initPhone();
            }
            dwDutyRoster = PhoneCache.phoneCache.get(time);
        }catch (Exception e){
            // e.printStackTrace();
            String phone=config.getStr("exceptWarnPhone","18838098890");
            dwDutyRoster.setFirstPhone(phone);
            dwDutyRoster.setSecondPhone(phone);
            dwDutyRoster.setThirdPhone(phone);
            dwDutyRoster.setFirstChargePerson("王晓东");
            dwDutyRoster.setSecondChargePerson("王晓东");
            dwDutyRoster.setThirdChargePerson("王晓东");
        }
        boolean warning=false;
        if(warningDate!=null){
            long between = DateUtil.between(warningDate, new Date(), DateUnit.MINUTE);
            if(between>=intervalMinute){
                warning=true;
            }
        }else {
            warning=true;
        }
        if(warning){
            String phone1 = dwDutyRoster.getFirstPhone();
            String phone2 = dwDutyRoster.getSecondPhone();
            String phone3 = dwDutyRoster.getThirdPhone();
            String alarmContent = "数据中台跑批节点数据跑批失败，请及时处理";
            if(StringUtils.isNotEmpty(content)){
                alarmContent=content;
            }
            log.info(alarmContent);
            log.info("第一顺序" + dwDutyRoster.getFirstChargePerson() + ":告警电话:" + phone1 + " 第二顺序" + dwDutyRoster.getSecondChargePerson() + ":告警电话：" + phone2 + " 第三顺序" + dwDutyRoster.getThirdChargePerson() + ":告警电话:" + phone3);
            String url = "https://aicc.cheryfs.cn/api/v1/config/openapi/contact/batch/json/upload";
            String token = "qD9w5lF8i8Dtn62mvxIRpaGP5FyrZiunGcoQB2JHKFuzB6T4BGwT2MqptoYECkPg";
            String dataSourceName = "system_alarm";
            String dataListElement = "" + phone1 + ",637839444jg76e72378,637839444jg76e72378,数据中台,智能外呼,跑批节点," + alarmContent + "," + phone2 + "," + phone3 + "";
            String abbreviate = "cheryfs";
            Map<String, String> headers = new HashMap<>();
            headers.put("token", token);
            headers.put("Content-Type", "application/json");
            JSONObject data = new JSONObject();
            data.put("dataSourceName", dataSourceName);
            JSONArray one = new JSONArray();
            String[] two = dataListElement.split(",");
            one.add(two);
            data.put("dataList", one);
            data.put("abbreviate", abbreviate);
            log.info("告警参数：" + JSON.toJSONString(data));
            HttpResponse response = HttpRequest.post(url)
                    .headerMap(headers, false)
                    .body(JSON.toJSONString(data))
                    .execute();
            log.info("告警结果返回:" + response);
            warningDate=new Date();
        }
    }

    public static List<InstanceEntityDto>  checkOracleScript(String bizDate,String endTime){
        List<InstanceEntityDto> list=new ArrayList<>();
        List<String> oracleScript=new ArrayList<>();
        Stream<String> stream=null;
        try {
            stream= Files.lines(Paths.get("back_oracle.txt"));
            oracleScript =stream.collect(Collectors.toList());
            for (String s:oracleScript){
                List<InstanceEntityDto> list1=new ArrayList<>();
                List<InstanceEntityDto> list2=getInstance(s.trim(),bizDate,endTime);
                log.info("查询结果:"+JSON.toJSONString(list2));
                list1=list2.stream()
                        .filter(instanceEntityDto -> !String.valueOf(DataWorksTaskStatus.SUCCESS.getCode()).equals(String.valueOf(instanceEntityDto.getStatus())))
                        .filter(instanceEntityDto -> !String.valueOf(DataWorksTaskStatus.FAILURE.getCode()).equals(String.valueOf(instanceEntityDto.getStatus())))
                        .collect(Collectors.toList());
                log.info("脚本:"+s);
                list.addAll(list1);
            }
            log.info("检测结果："+JSON.toJSONString(list));
        }catch (Exception e){
            e.printStackTrace();
        }finally {
            if(stream!=null){
                stream.close();
            }
        }
        return list;
    }


    public static String buildUrlWithParams(String baseUrl, Map<String, String> params) {
        StringBuilder urlBuilder = new StringBuilder(baseUrl);
        if (!params.isEmpty()) {
            urlBuilder.append("?");
            boolean firstParam = true;
            for (Map.Entry<String, String> entry : params.entrySet()) {
                if (!firstParam) {
                    urlBuilder.append("&");
                }
                urlBuilder.append(entry.getKey()).append("=").append(entry.getValue());
                firstParam = false;
            }
        }
        return urlBuilder.toString();
    }
    public static void phoneWarning(String content,String firstPhone,String secondPhone,String thirdPhone) {

        String alarmContent = "数据中台跑批节点数据跑批失败，请及时处理";
        if(StringUtils.isNotEmpty(content)){
            alarmContent=content;
        }
        log.info(alarmContent);
        log.info("第一顺序" + firstPhone + ":告警电话:" + firstPhone + " 第二顺序" +secondPhone + ":告警电话：" + secondPhone + " 第三顺序" + thirdPhone + ":告警电话:" + thirdPhone);
        String url = "https://aicc.cheryfs.cn/api/v1/config/openapi/contact/batch/json/upload";
        String token = "qD9w5lF8i8Dtn62mvxIRpaGP5FyrZiunGcoQB2JHKFuzB6T4BGwT2MqptoYECkPg";
        String dataSourceName = "system_alarm";
        String dataListElement = "" + firstPhone + ",637839444jg76e72378,637839444jg76e72378,数据中台,智能外呼,跑批节点," + alarmContent + "," + secondPhone + "," + thirdPhone + "";
        String abbreviate = "cheryfs";
        Map<String, String> headers = new HashMap<>();
        headers.put("token", token);
        headers.put("Content-Type", "application/json");
        JSONObject data = new JSONObject();
        data.put("dataSourceName", dataSourceName);
        JSONArray one = new JSONArray();
        String[] two = dataListElement.split(",");
        one.add(two);
        data.put("dataList", one);
        data.put("abbreviate", abbreviate);
        log.info("告警参数：" + JSON.toJSONString(data));
        HttpResponse response = HttpRequest.post(url)
                .headerMap(headers, false)
                .body(JSON.toJSONString(data))
                .execute();
        log.info("告警结果返回:" + response);
    }

    public static void grantTable(){
        try {
            List<String>  tableList=new ArrayList<>();
            List<String> tend=new ArrayList<>();
            File file = new File("table.json");
            tend=Files.lines(Paths.get("tend.txt")).collect(Collectors.toList());
            if(!file.exists()){
                tableList= Files.lines(Paths.get("table.txt")).collect(Collectors.toList());
            }else{
                String table=readerMethod(file);
                JSONObject jsonObject=JSON.parseObject(table);
                JSONArray jsonArray=jsonObject.getJSONArray("model");
                for (int i=0;i<jsonArray.size();i++){
                    JSONObject jsonObject1=jsonArray.getJSONObject(i);
                    tableList.add(jsonObject1.getString("name"));
                }
            }
            File files = new File("cookie.txt");
            int batchSize = 30;
            List<List<String>> batches = partition(tableList, batchSize);
            for (int i=0;i<batches.size();i++){
                List<String> tbs=batches.get(i);
                List<String> guid=new ArrayList<>();
                for (int j=0;j<tbs.size();j++){
                    guid.add("odps.datawarehouse_pro");
                }
                if(!Files.exists(Paths.get("applyed.txt"))){
                    applyList=new ArrayList<>();
                }else{
                    applyList= Files.lines(Paths.get("applyed.txt")).collect(Collectors.toList());
                }
                if(applyList.containsAll(tbs)){
                    continue;
                }
                JSONArray action=new JSONArray();
                action.add("Select");
                action.add("Describe");
                action.add("All");
                String url="https://guard.res.cloud.cheryfs.cn/meta/project/listTableColumn?projectName=DatawareHouse_Pro&workspace=10022&env=1&pageSize=1&current=1";
                url+="&tableName="+JSON.toJSONString(tbs)+"&projectGuid="+JSON.toJSONString(guid);
                String cookies=readerMethod(files);
                if(StringUtils.isEmpty(cookies)){
                    login();
                }
                String urls= HttpUtil.encodeParams(url, Charset.defaultCharset()).replaceAll(",","%2C");
                String result3= HttpRequest.get(urls)
                        .cookie(cookies)
                        .execute().body();
            //    log.info("查询数据结果"+result3);
                JSONObject jsonObject=JSON.parseObject(result3);
                if(jsonObject.getString("success").equals("true")){
                    JSONArray jsonArray=jsonObject.getJSONArray("model");
                    JSONObject param=new JSONObject();
                    param.put("applyReason","查询数据");
                    param.put("deadlineDate","forever");
                    param.put("deadlineType","forever");
                    JSONArray granteeObjectList=new JSONArray();
                    JSONObject grant=new JSONObject();
                    String grantId="5242897009517417416";
                    String grantType="1";
                    String granteeTypeSub="103";
                    if(tend.size()>0){
                        grantId=tend.get(0);
                        grantType=tend.get(1);
                        granteeTypeSub=tend.get(2);
                    }
                    grant.put("granteeId",grantId);
                    grant.put("granteeType",grantType);
                    grant.put("granteeTypeSub",granteeTypeSub);
                    granteeObjectList.add(grant);
                    param.put("applyReason","查询数据");
                    param.put("granteeObjectList",granteeObjectList);
                    JSONObject  projectMeta=new JSONObject();
                    projectMeta.put("projectName","DatawareHouse_Pro");
                    projectMeta.put("envCode",1);
                    projectMeta.put("projectId","10022");
                    JSONArray  objectMetaList=new JSONArray();
                    for (int m=0;m<jsonArray.size();m++){
                        JSONObject jsonObject1=jsonArray.getJSONObject(m);
                        JSONObject one=new JSONObject();
                        one.put("action",action);
                        one.put("name",jsonObject1.getString("name"));
                        one.put("projectGuid",jsonObject1.getString("projectGuid"));
                        JSONArray jsonArray1=jsonObject1.getJSONArray("columnMetaList");
                        JSONArray columnMetaList=new JSONArray();
                        for (int s=0;s<jsonArray1.size();s++){
                            JSONObject jsonObject2=jsonArray1.getJSONObject(s);
                            JSONObject one1=new JSONObject();
                            one1.put("name",jsonObject2.getString("name"));
                            columnMetaList.add(one1);
                        }
                        one.put("columnMetaList",columnMetaList);
                        objectMetaList.add(one);
                    }
                    projectMeta.put("objectMetaList",objectMetaList);
                    param.put("projectMeta",projectMeta);
         //          log.info("申请参数"+param.toJSONString());
                    String result= HttpRequest.post("https://guard.res.cloud.cheryfs.cn/permission/apply")
                            .cookie(cookies)
                            .body(param.toJSONString())
                            .execute().body();
             //       log.info("申请"+result);
                    JSONObject cc=JSON.parseObject(result);
                    if(cc.getString("success").equals("true")){
                        String aproveUrl="https://guard.res.cloud.cheryfs.cn/permission/approve/listTodo?current=1&10&formType=waitingForApprove";
                        String result4= HttpRequest.get(aproveUrl)
                                .cookie(cookies)
                                .execute().body();
                 //       log.info("待审核"+result4);
                        JSONObject mm=JSON.parseObject(result4);
                        if(mm.getString("success").equals("true")){
                            JSONArray dd=mm.getJSONArray("model");
                            JSONObject app=new JSONObject();
                            app.put("action","1");
                            app.put("comment","同意");
                            JSONArray flowIds=new JSONArray();
                            for (int k=0;k<dd.size();k++){
                                JSONObject jsonObject1=dd.getJSONObject(k);
                                String flowId=jsonObject1.getString("flowId");
                                flowIds.add(flowId);
                            }
                            app.put("flowIds",flowIds);
                            String result5= HttpRequest.post("https://guard.res.cloud.cheryfs.cn/permission/approve/submit/batch")
                                    .cookie(cookies)
                                    .header(Header.CONTENT_TYPE, "application/json")
                                    .body(app.toJSONString())
                                    .execute().body();
                      //      log.info("审批参数"+app.toJSONString());
                            log.info("审批"+result5);
                            Files.write(Paths.get("applyed.txt"),tbs, StandardOpenOption.CREATE,StandardOpenOption.APPEND);
                        }
                    }
                }
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
    public static <T> List<List<T>> partition(List<T> list, int size) {
        List<List<T>> partitions = new ArrayList<>();
        for (int i = 0; i < list.size(); i += size) {
            partitions.add(new ArrayList<>(list.subList(i, Math.min(i + size, list.size()))));
        }
        return partitions;
    }

    public static Long getMetaDBTableList(int page){
        Long total=1L;
        GetMetaDBTableListRequest request = new GetMetaDBTableListRequest();
        request.setPageSize(100);
        request.setPageNumber(page);
        request.setDataSourceType("odps");
        request.setAppGuid("odps.datawarehouse_pro");
        StringBuffer sb=new StringBuffer();
        StringBuffer sbNoCommment=new StringBuffer();
            GetMetaDBTableListResponse response = null;
            try {
                response = client.getAcsResponse(request);
                if(response.getData()!=null){
                    total=response.getData().getTotalCount();
                    List<GetMetaDBTableListResponse.Data.TableEntityListItem> tableEntityListItems=response.getData().getTableEntityList();
                    for (int i=0;i<tableEntityListItems.size();i++){
                        GetMetaDBTableListResponse.Data.TableEntityListItem tableEntityListItem=tableEntityListItems.get(i);
                    //    log.info("表名"+tableEntityListItem.getTableName());
                        GetMetaTableFullInfoRequest getMetaTableFullInfoRequest=new GetMetaTableFullInfoRequest();
                        getMetaTableFullInfoRequest.setTableGuid(tableEntityListItem.getTableGuid());
                        GetMetaTableFullInfoResponse getMetaTableFullInfoResponse=client.getAcsResponse(getMetaTableFullInfoRequest);
                        String  comment="";
                        if(getMetaTableFullInfoResponse.getData()!=null){
                           comment=getMetaTableFullInfoResponse.getData().getComment();
                        }
                        log.info("TableGuid"+tableEntityListItem.getTableGuid()+"  "+comment);
                        sb.append(tableEntityListItem.getTableName()+"  "+comment+"\n");
                        sbNoCommment.append(tableEntityListItem.getTableName()+"\n");
                    }
                }
            } catch (ClientException e) {
                throw new RuntimeException(e);
            }catch (Exception e){
                e.printStackTrace();
            }finally {
                try {
                    Files.write(Paths.get("table.txt"),sbNoCommment.toString().getBytes(), StandardOpenOption.CREATE,StandardOpenOption.TRUNCATE_EXISTING);
                    Files.write(Paths.get("tableAndComments.txt"),sb.toString().getBytes(), StandardOpenOption.APPEND,StandardOpenOption.APPEND);
                } catch (IOException e) {
                    throw new RuntimeException(e);
                }
            }
            return total;
    }


    public static MultiSheetExcelExporter.SheetMeta getMetaTableColumn(String table){
         MultiSheetExcelExporter.SheetMeta sheetMeta=new MultiSheetExcelExporter.SheetMeta();
        GetMetaTableFullInfoRequest request = new GetMetaTableFullInfoRequest();
        request.setPageSize(100);
        request.setTableGuid(table);
        request.setDataSourceType("odps");
        GetMetaTableFullInfoResponse response = null;
        try {
            response = client.getAcsResponse(request);
            log.info(JSON.toJSONString(response));
            if(response.getData()!=null){
               List<GetMetaTableFullInfoResponse.Data.ColumnListItem> columnList=response.getData().getColumnList();
               Collections.sort(columnList,Comparator.comparingInt(GetMetaTableFullInfoResponse.Data.ColumnListItem::getPosition));
               String tableName=response.getData().getTableName();
               String tableComment=response.getData().getComment();
                List<Map<String, String>> sheetData = new ArrayList<>();
               for (int i=0;i<columnList.size();i++){
                   GetMetaTableFullInfoResponse.Data.ColumnListItem columnListItem=columnList.get(i);
                   String comment=columnListItem.getComment();
                   String columnType=columnListItem.getColumnType();
                   String columnName=columnListItem.getColumnName();
                   if(columnName.equals("ds")){
                       continue;
                   }
                   GetMetaColumnLineageRequest getMetaColumnLineageRequest =new GetMetaColumnLineageRequest();
                   getMetaColumnLineageRequest.setColumnGuid(columnListItem.getColumnGuid());
                   getMetaColumnLineageRequest.setDirection("up");
                   GetMetaColumnLineageResponse getMetaColumnLineageResponse=client.getAcsResponse(getMetaColumnLineageRequest);
                   log.info("血缘:"+JSON.toJSONString(getMetaColumnLineageResponse));
                   if(getMetaColumnLineageResponse.getData()!=null){
                       List<GetMetaColumnLineageResponse.Data.DataEntityListItem> lineageList=getMetaColumnLineageResponse.getData().getDataEntityList();
                        if(lineageList.size()<=0){
                            sheetData.add(createFieldMap(columnName,comment,columnType,"","","",""));
                        }
                       for (int j=0;j<lineageList.size();j++){
                           GetMetaColumnLineageResponse.Data.DataEntityListItem dataEntityListItem=lineageList.get(j);
                           log.info("来源字段");
                           log.info(dataEntityListItem.getTableName());
                           log.info(dataEntityListItem.getColumnName());
                           String sourceTableName=dataEntityListItem.getTableName();
                           String sourceColumnName=dataEntityListItem.getColumnName();
                           if(!tableFullMap.containsKey(sourceTableName)){
                               getTableFullInfo("odps.datawarehouse_pro."+sourceTableName);
                           }
                           String sourceTableNameCommnet=tableFullMap.get(sourceTableName);
                           String sourceColumnNameCommnet=tableFullMap.get(sourceTableName+sourceColumnName);
                           sheetData.add(createFieldMap(columnName,comment,columnType,sourceTableName,sourceTableNameCommnet,sourceColumnName,sourceColumnNameCommnet));
                           log.info("字段信息"+JSON.toJSONString(sheetData));
                       }
                   }
               }
                sheetMeta.setSheetName(tableName);
                sheetMeta.setChineseName(tableComment);
                sheetMeta.setFields(sheetData);
                sheetMeta.setRemark("");
               log.info("获取表字段信息"+JSON.toJSONString(sheetMeta));
            }
        } catch (ClientException e) {
            throw new RuntimeException(e);
        }catch (Exception e){
            e.printStackTrace();
        }
        return sheetMeta;
    }


    public static String getInstanceLog(Long instanceId){
        String errorLog=null;
        GetInstanceLogRequest request = new GetInstanceLogRequest();
        request.setInstanceId(instanceId);
        request.setProjectEnv("PROD");
        GetInstanceLogResponse response = null;
        try {
            try {
                response = client.getAcsResponse(request);
            //    log.info(JSON.toJSONString(response));
                if(response.getHttpStatusCode()==200){
                    errorLog=response.getData();
                }
            } catch (ClientException e) {
                throw new RuntimeException(e);
            }catch (Exception e){
                e.printStackTrace();
            }

        }catch (Exception e){
            e.printStackTrace();
        }
        return errorLog;
    }

    public  static void insertInstanceLog(String bizDate,List<Record> records,String tableName){
            Account account = new AliyunAccount("feupFUuHYOxrohjn", "MUFrk9HsB94LXzyDZZA0lhPvvoa1J0");
            Odps odps = new Odps(account);
            String odpsUrl = "http://service.cn-wuhu-qrhy-d01.odps.ops.cloud.cheryfs.cn/api";
            odps.setEndpoint(odpsUrl);
            odps.setDefaultProject("datawarehouse_pro");
           TableTunnel tunnel = new TableTunnel(odps);
        PartitionSpec partitionSpec=new PartitionSpec();
        partitionSpec.set("ds",bizDate);
        if(StringUtils.isEmpty(tableName)){
            tableName="dw_instance_task_info_log";
        }
        Table table = odps.tables().get(tableName);//获取当前表
        boolean a= false;//判断上述定义分区在表中是否存在
        try {
            a = table.hasPartition(partitionSpec);
            if(!a){
                table.createPartition(partitionSpec);
            }
        } catch (OdpsException e) {
            throw new RuntimeException(e);
        }
//        if(a){
//           log.info("分区已经存在，可以直接上传数据");
//        }else{
//            log.info("分区不存在，先创建分区再上传数据");
//            try {
//                table.createPartition(partitionSpec);
//            } catch (OdpsException e) {
//                throw new RuntimeException(e);
//            }
//        }

        try {
            TableTunnel.UploadSession uploadSession = (TableTunnel.UploadSession) tunnel.createUploadSession(
                 "datawarehouse_pro",
                 tableName,
                 partitionSpec
         );
            RecordWriter rw = uploadSession.openRecordWriter(0);
            for (Record record : records){
                rw.write(record);
            }
            rw.close();
         //   Long[] blocks = uploadSession.getBlockList();
            uploadSession.commit(new Long[]{0L});
        } catch (TunnelException | IOException e) {
            throw new RuntimeException(e);
        }
    }


    public static List<Record>  getRecords(int page,String bizDate) {
        // 创建DefaultAcsClient实例并初始化
        Date newDate=null;
        if(StringUtils.isEmpty(bizDate)){
             newDate = DateUtil.offset(new Date(), DateField.DAY_OF_MONTH, -1);
            bizDate=DateUtil.format(newDate, "yyyyMMdd");
        }else {
            newDate = DateUtil.parse(bizDate, "yyyyMMdd");
        }
        String startTime= DateUtil.format(newDate, "yyyy-MM-dd")+" 00:00:00";
        String endTime= DateUtil.format(newDate, "yyyy-MM-dd")+" 23:59:59";
        // 创建API请求并设置参数
        ListInstancesRequest request = new ListInstancesRequest();
        request.setProjectId(10022L); // 替换为你的DataWorks项目名称
        request.setProjectEnv("PROD");
        request.setBeginBizdate(startTime);
        request.setEndBizdate(endTime);
        request.setPageSize(100);
        request.setPageNumber(page);
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        List<Record> instances=new ArrayList<>();
        // 可以根据需要设置更多参数，如pageNumber、pageSize等
        try {
            // 调用ListInstances方法
            ListInstancesResponse response = client.getAcsResponse(request);
            // 处理响应结果
            if (response != null && response.getSuccess()) {
                JobVarUtil.totalCount=response.getData().getTotalCount();
                List<ListInstancesResponse.Data.Instance> dataInstances=response.getData().getInstances();
                log.info("当前页数:"+page);
                if(dataInstances!=null&&dataInstances.size()>0){
                    log.info("获取实例信息:"+JSON.toJSONString(response.getData()));
                    for (ListInstancesResponse.Data.Instance instance : dataInstances){
                        Long instaceId=instance.getInstanceId();
                        if(JobVarUtil.instanceMap.containsKey(String.valueOf(instaceId))){
                            continue;
                        }
                        if(instance.getNodeName().equals("ods_ccm_loan_vehicle_info_push")){
                            log.info("当前页数:"+page);
                            log.info("打印ods_ccm_loan_vehicle_info_push:"+instance.getNodeName());
                        }
                      //  if(instance.getStatus().equals(Instance.TaskStatus.Status.SUCCESS.name())||instance.getStatus().equals(Instance.TaskStatus.Status.FAILED.name())){
                            Column[] columns1=new Column[36];
                            columns1[0]=(new Column("nodeId", OdpsType.valueOf("BIGINT")));
                            columns1[1]=(new Column("instanceId", OdpsType.valueOf("BIGINT")));
                            columns1[2]=(new Column("dagId", OdpsType.valueOf("BIGINT")));
                            columns1[3]=(new Column("dagType", OdpsType.valueOf("STRING")));
                            columns1[4]=(new Column("status", OdpsType.valueOf("STRING")));
                            columns1[5]=(new Column("bizdate", OdpsType.valueOf("STRING")));
                            columns1[6]=(new Column("cycTime", OdpsType.valueOf("TIMESTAMP")));
                            columns1[7]=(new Column("createTime", OdpsType.valueOf("TIMESTAMP")));
                            columns1[8]=(new Column("modifyTime", OdpsType.valueOf("TIMESTAMP")));
                            columns1[9]=(new Column("nodeName", OdpsType.valueOf("STRING")));
                            columns1[10]=(new Column("beginWaitTimeTime", OdpsType.valueOf("TIMESTAMP")));
                            columns1[11]=(new Column("beginWaitResTime", OdpsType.valueOf("TIMESTAMP")));
                            columns1[12]=(new Column("beginRunningTime", OdpsType.valueOf("TIMESTAMP")));
                            columns1[13]=(new Column("paramValues", OdpsType.valueOf("STRING")));
                            columns1[14]=(new Column("finishTime", OdpsType.valueOf("TIMESTAMP")));
                            columns1[15]=(new Column("priority", OdpsType.valueOf("BIGINT")));
                            columns1[16]=(new Column("baselineId", OdpsType.valueOf("BIGINT")));
                            columns1[17]=(new Column("repeatability", OdpsType.valueOf("BOOLEAN")));
                            columns1[18]=(new Column("repeatInterval", OdpsType.valueOf("BIGINT")));
                            columns1[19]=(new Column("connection", OdpsType.valueOf("STRING")));
                            columns1[20]=(new Column("dqcType", OdpsType.valueOf("BIGINT")));
                            columns1[21]=(new Column("dqcDescription", OdpsType.valueOf("STRING")));
                            columns1[22]=(new Column("errorMessage", OdpsType.valueOf("STRING")));
                            columns1[23]=(new Column("relatedFlowId", OdpsType.valueOf("BIGINT")));
                            columns1[24]=(new Column("taskType", OdpsType.valueOf("STRING")));
                            columns1[25]=(new Column("taskRerunTime", OdpsType.valueOf("BIGINT")));
                            columns1[26]=(new Column("businessId", OdpsType.valueOf("BIGINT")));
                            columns1[27]=(new Column("waitSchTime", OdpsType.valueOf("BIGINT")));
                            columns1[28]=(new Column("waitSchTimeDesc", OdpsType.valueOf("STRING")));
                            columns1[29]=(new Column("waitResTime", OdpsType.valueOf("BIGINT")));
                            columns1[30]=(new Column("waitResTimeDesc", OdpsType.valueOf("STRING")));
                            columns1[31]=(new Column("runTime", OdpsType.valueOf("BIGINT")));
                            columns1[32]=(new Column("runTimeDesc", OdpsType.valueOf("STRING")));
                            columns1[33]=(new Column("totalBytes", OdpsType.valueOf("STRING")));
                            columns1[34]=(new Column("totalNum", OdpsType.valueOf("STRING")));
                            columns1[35]=(new Column("error_records", OdpsType.valueOf("STRING")));
                            Record record=new ArrayRecord(columns1);
                            record.setBigint(0,instance.getNodeId());
                            record.setBigint(1,instance.getInstanceId());
                            record.setBigint(2,instance.getDagId());
                            record.setString(3,instance.getDagType());
                            record.setString(4,instance.getStatus());
                            record.setString(5, sdf.format(new Date(instance.getBizdate())));
                            record.setDatetime(6,new Timestamp(instance.getCycTime()));
                            record.setDatetime(7,new Timestamp(instance.getCreateTime()));
                            record.setDatetime(8,new Timestamp(instance.getModifyTime()));
                            record.setString(9,instance.getNodeName());
                            if(instance.getBeginWaitTimeTime()!=null){
                                record.setDatetime(10,new Timestamp(instance.getBeginWaitTimeTime()));
                            }else {
                                record.setDatetime(10,null);
                            }
                            if (instance.getBeginWaitResTime()!=null){
                                record.setDatetime(11,new Timestamp(instance.getBeginWaitResTime()));
                            }else {
                                record.setDatetime(11,null);
                            }
                            if(instance.getBeginRunningTime()!=null){
                                record.setDatetime(12,new Timestamp(instance.getBeginRunningTime()));
                            }else {
                                record.setDatetime(12,null);
                            }
                            record.setString(13,instance.getParamValues());
                            if(instance.getFinishTime()!=null){
                                record.setDatetime(14,new Timestamp(instance.getFinishTime()));
                            }else {
                                record.setDatetime(14,null);
                            }
                            record.setBigint(15,instance.getPriority().longValue());
                            record.setBigint( 16,instance.getBaselineId());
                            record.setBoolean(17,instance.getRepeatability());
                            record.setBigint(18,instance.getRepeatInterval());
                            record.setString(19,instance.getConnection());
                            if(instance.getDqcType()!=null){
                                record.setBigint(20,instance.getDqcType().longValue());
                            }else{
                                record.setBigint(20,0L);
                            }
                            record.setString(21,instance.getDqcDescription());
                            record.setString(22,instance.getErrorMessage());
                            record.setBigint(23,instance.getRelatedFlowId());
                            record.setString(24,instance.getTaskType());
                            if(instance.getTaskRerunTime()!=null){
                                record.setBigint(25,instance.getTaskRerunTime().longValue());
                            }else {
                                record.setBigint(25,0L);
                            }
                            record.setBigint(26,instance.getBaselineId());
                            if(instance.getBeginWaitTimeTime()!=null&&instance.getBeginWaitResTime()!=null){
                                Long secods = DateUtil.between(new Date(instance.getBeginWaitTimeTime()), new Date(instance.getBeginWaitResTime()), DateUnit.SECOND);
                                Long ms = DateUtil.between(new Date(instance.getBeginWaitTimeTime()), new Date(instance.getBeginWaitResTime()), DateUnit.MS);
                                record.setBigint(27,secods);
                                String formatBetween = DateUtil.formatBetween(ms, BetweenFormatter.Level.SECOND);
                                record.setString(28,formatBetween);
                            }
                            if(instance.getBeginWaitResTime()!=null&&instance.getBeginRunningTime()!=null){
                                Long secods = DateUtil.between(new Date(instance.getBeginWaitResTime()), new Date(instance.getBeginRunningTime()), DateUnit.SECOND);
                                Long ms = DateUtil.between(new Date(instance.getBeginWaitResTime()), new Date(instance.getBeginRunningTime()), DateUnit.MS);
                                record.setBigint(29,secods);
                                String formatBetween = DateUtil.formatBetween(ms, BetweenFormatter.Level.SECOND);
                                record.setString(30,formatBetween);
                            }
                            if(instance.getBeginRunningTime()!=null&&instance.getFinishTime()!=null){
                                Long secods = DateUtil.between(new Date(instance.getBeginRunningTime()), new Date(instance.getFinishTime()), DateUnit.SECOND);
                                Long ms = DateUtil.between(new Date(instance.getBeginRunningTime()), new Date(instance.getFinishTime()), DateUnit.MS);
                                record.setBigint(31,secods);
                                String formatBetween = DateUtil.formatBetween(ms, BetweenFormatter.Level.SECOND);
                                record.setString(32,formatBetween);
                            }

                            if(instanceSyncTaskMap.containsKey(instance.getNodeName())){
                                JSONObject jsonObject=instanceSyncTaskMap.get(instance.getNodeName());
                                record.setString(33,jsonObject.getString("totalBytes"));
                                record.setString(34,jsonObject.getString("totalRecords"));
                                record.setString(35,jsonObject.getString("errorRecords"));
                            }
                            instances.add(record);
                           mailRecords.add(record);
                        JobVarUtil.instanceMap.put(String.valueOf(instaceId),String.valueOf(instaceId));
                       // }
                    }
                }

            } else {
                System.out.println("Failed to list instances. Error code: " + response.getErrorCode() + ", Error message: " + response.getErrorMessage());
            }
        } catch (ClientException e) {
            e.printStackTrace();
        }
        return instances;
    }

    public static String getNodeCode(String nodeId){
        GetNodeCodeRequest request = new GetNodeCodeRequest();
        request.setNodeId(Long.valueOf(nodeId));
        request.setProjectEnv("PROD");
        String res="";
        try {
            GetNodeCodeResponse response = client.getAcsResponse(request);
            if (response.getSuccess()) {
                res=response.getData();
            } else {
                System.out.println("Failed to get node code. Error code: " + response.getErrorCode() + ", Error message: " + response.getErrorMessage());
            }
        }catch (Exception e) {
            throw new RuntimeException(e);
        }
        return res;
    }

    public static void getInstanceByNodeId(String nodeId){
        GetInstanceRequest request = new GetInstanceRequest();
        request.setInstanceId(Long.valueOf(nodeId));
        request.setProjectEnv("PROD");
        String res="";
        try {
            GetInstanceResponse response = client.getAcsResponse(request);
            if (response.getSuccess()) {
                log.info("Node code: " + JSON.toJSONString(response.getData()));
            } else {
                System.out.println("Failed to get node code. Error code: " + response.getErrorCode() + ", Error message: " + response.getErrorMessage());
            }
        }catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    public static List<Record>  getRecords(String bizDate) {
        // 创建DefaultAcsClient实例并初始化
        Date newDate=null;
        if(StringUtils.isEmpty(bizDate)){
            newDate = DateUtil.offset(new Date(), DateField.DAY_OF_MONTH, -1);
            bizDate=DateUtil.format(newDate, "yyyyMMdd");
        }else {
            newDate = DateUtil.parse(bizDate, "yyyyMMdd");
        }
        String startTime= DateUtil.format(newDate, "yyyy-MM-dd")+" 00:00:00";
        String endTime= DateUtil.format(newDate, "yyyy-MM-dd")+" 23:59:59";
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        List<Record> instances=new ArrayList<>();
        Map<String,ListInstancesResponse.Data.Instance> cfMap=new HashMap<>();
        int page=0;
        for (int i=1;i<30;i++){
            // 创建API请求并设置参数
            ListInstancesRequest request = new ListInstancesRequest();
            request.setProjectId(10022L); // 替换为你的DataWorks项目名称
            request.setProjectEnv("PROD");
            request.setBeginBizdate(startTime);
            request.setEndBizdate(endTime);
            request.setPageSize(100);
            request.setPageNumber(i);
            // 可以根据需要设置更多参数，如pageNumber、pageSize等
            try {
                // 调用ListInstances方法
                ListInstancesResponse response = client.getAcsResponse(request);
                log.info("请求参数："+JSON.toJSONString(request));
                // 处理响应结果
                if (response != null && response.getSuccess()) {
                    JobVarUtil.totalCount=response.getData().getTotalCount();
                    List<ListInstancesResponse.Data.Instance> dataInstances=response.getData().getInstances();
                    log.info("当前页数:"+page);
                    if(dataInstances!=null&&dataInstances.size()>0){
                        log.info("获取实例信息:"+JSON.toJSONString(response.getData()));
                        for (ListInstancesResponse.Data.Instance instance : dataInstances){
                            Long instaceId=instance.getInstanceId();
                            if(JobVarUtil.instanceMap.containsKey(String.valueOf(instaceId))){
                                cfMap.put(instance.getNodeName(),instance);
                                continue;
                            }
                            if(instance.getNodeName().equals("ods_ccm_loan_vehicle_info_push")){
                                log.info("当前页数:"+page);
                                log.info("打印ods_ccm_loan_vehicle_info_push:"+instance.getNodeName());
                            }
                            //  if(instance.getStatus().equals(Instance.TaskStatus.Status.SUCCESS.name())||instance.getStatus().equals(Instance.TaskStatus.Status.FAILED.name())){
                            Column[] columns1=new Column[36];
                            columns1[0]=(new Column("nodeId", OdpsType.valueOf("BIGINT")));
                            columns1[1]=(new Column("instanceId", OdpsType.valueOf("BIGINT")));
                            columns1[2]=(new Column("dagId", OdpsType.valueOf("BIGINT")));
                            columns1[3]=(new Column("dagType", OdpsType.valueOf("STRING")));
                            columns1[4]=(new Column("status", OdpsType.valueOf("STRING")));
                            columns1[5]=(new Column("bizdate", OdpsType.valueOf("STRING")));
                            columns1[6]=(new Column("cycTime", OdpsType.valueOf("TIMESTAMP")));
                            columns1[7]=(new Column("createTime", OdpsType.valueOf("TIMESTAMP")));
                            columns1[8]=(new Column("modifyTime", OdpsType.valueOf("TIMESTAMP")));
                            columns1[9]=(new Column("nodeName", OdpsType.valueOf("STRING")));
                            columns1[10]=(new Column("beginWaitTimeTime", OdpsType.valueOf("TIMESTAMP")));
                            columns1[11]=(new Column("beginWaitResTime", OdpsType.valueOf("TIMESTAMP")));
                            columns1[12]=(new Column("beginRunningTime", OdpsType.valueOf("TIMESTAMP")));
                            columns1[13]=(new Column("paramValues", OdpsType.valueOf("STRING")));
                            columns1[14]=(new Column("finishTime", OdpsType.valueOf("TIMESTAMP")));
                            columns1[15]=(new Column("priority", OdpsType.valueOf("BIGINT")));
                            columns1[16]=(new Column("baselineId", OdpsType.valueOf("BIGINT")));
                            columns1[17]=(new Column("repeatability", OdpsType.valueOf("BOOLEAN")));
                            columns1[18]=(new Column("repeatInterval", OdpsType.valueOf("BIGINT")));
                            columns1[19]=(new Column("connection", OdpsType.valueOf("STRING")));
                            columns1[20]=(new Column("dqcType", OdpsType.valueOf("BIGINT")));
                            columns1[21]=(new Column("dqcDescription", OdpsType.valueOf("STRING")));
                            columns1[22]=(new Column("errorMessage", OdpsType.valueOf("STRING")));
                            columns1[23]=(new Column("relatedFlowId", OdpsType.valueOf("BIGINT")));
                            columns1[24]=(new Column("taskType", OdpsType.valueOf("STRING")));
                            columns1[25]=(new Column("taskRerunTime", OdpsType.valueOf("BIGINT")));
                            columns1[26]=(new Column("businessId", OdpsType.valueOf("BIGINT")));
                            columns1[27]=(new Column("waitSchTime", OdpsType.valueOf("BIGINT")));
                            columns1[28]=(new Column("waitSchTimeDesc", OdpsType.valueOf("STRING")));
                            columns1[29]=(new Column("waitResTime", OdpsType.valueOf("BIGINT")));
                            columns1[30]=(new Column("waitResTimeDesc", OdpsType.valueOf("STRING")));
                            columns1[31]=(new Column("runTime", OdpsType.valueOf("BIGINT")));
                            columns1[32]=(new Column("runTimeDesc", OdpsType.valueOf("STRING")));
                            columns1[33]=(new Column("totalBytes", OdpsType.valueOf("STRING")));
                            columns1[34]=(new Column("totalNum", OdpsType.valueOf("STRING")));
                            columns1[35]=(new Column("error_records", OdpsType.valueOf("STRING")));
                            Record record=new ArrayRecord(columns1);
                            record.setBigint(0,instance.getNodeId());
                            record.setBigint(1,instance.getInstanceId());
                            record.setBigint(2,instance.getDagId());
                            record.setString(3,instance.getDagType());
                            record.setString(4,instance.getStatus());
                            record.setString(5, sdf.format(new Date(instance.getBizdate())));
                            record.setDatetime(6,new Timestamp(instance.getCycTime()));
                            record.setDatetime(7,new Timestamp(instance.getCreateTime()));
                            record.setDatetime(8,new Timestamp(instance.getModifyTime()));
                            record.setString(9,instance.getNodeName());
                            if(instance.getBeginWaitTimeTime()!=null){
                                record.setDatetime(10,new Timestamp(instance.getBeginWaitTimeTime()));
                            }else {
                                record.setDatetime(10,null);
                            }
                            if (instance.getBeginWaitResTime()!=null){
                                record.setDatetime(11,new Timestamp(instance.getBeginWaitResTime()));
                            }else {
                                record.setDatetime(11,null);
                            }
                            if(instance.getBeginRunningTime()!=null){
                                record.setDatetime(12,new Timestamp(instance.getBeginRunningTime()));
                            }else {
                                record.setDatetime(12,null);
                            }
                            record.setString(13,instance.getParamValues());
                            if(instance.getFinishTime()!=null){
                                record.setDatetime(14,new Timestamp(instance.getFinishTime()));
                            }else {
                                record.setDatetime(14,null);
                            }
                            record.setBigint(15,instance.getPriority().longValue());
                            record.setBigint( 16,instance.getBaselineId());
                            record.setBoolean(17,instance.getRepeatability());
                            record.setBigint(18,instance.getRepeatInterval());
                            record.setString(19,instance.getConnection());
                            if(instance.getDqcType()!=null){
                                record.setBigint(20,instance.getDqcType().longValue());
                            }else{
                                record.setBigint(20,0L);
                            }
                            record.setString(21,instance.getDqcDescription());
                            record.setString(22,instance.getErrorMessage());
                            record.setBigint(23,instance.getRelatedFlowId());
                            record.setString(24,instance.getTaskType());
                            if(instance.getTaskRerunTime()!=null){
                                record.setBigint(25,instance.getTaskRerunTime().longValue());
                            }else {
                                record.setBigint(25,0L);
                            }
                            record.setBigint(26,instance.getBaselineId());
                            if(instance.getBeginWaitTimeTime()!=null&&instance.getBeginWaitResTime()!=null){
                                Long secods = DateUtil.between(new Date(instance.getBeginWaitTimeTime()), new Date(instance.getBeginWaitResTime()), DateUnit.SECOND);
                                Long ms = DateUtil.between(new Date(instance.getBeginWaitTimeTime()), new Date(instance.getBeginWaitResTime()), DateUnit.MS);
                                record.setBigint(27,secods);
                                String formatBetween = DateUtil.formatBetween(ms, BetweenFormatter.Level.SECOND);
                                record.setString(28,formatBetween);
                            }
                            if(instance.getBeginWaitResTime()!=null&&instance.getBeginRunningTime()!=null){
                                Long secods = DateUtil.between(new Date(instance.getBeginWaitResTime()), new Date(instance.getBeginRunningTime()), DateUnit.SECOND);
                                Long ms = DateUtil.between(new Date(instance.getBeginWaitResTime()), new Date(instance.getBeginRunningTime()), DateUnit.MS);
                                record.setBigint(29,secods);
                                String formatBetween = DateUtil.formatBetween(ms, BetweenFormatter.Level.SECOND);
                                record.setString(30,formatBetween);
                            }
                            if(instance.getBeginRunningTime()!=null&&instance.getFinishTime()!=null){
                                Long secods = DateUtil.between(new Date(instance.getBeginRunningTime()), new Date(instance.getFinishTime()), DateUnit.SECOND);
                                Long ms = DateUtil.between(new Date(instance.getBeginRunningTime()), new Date(instance.getFinishTime()), DateUnit.MS);
                                record.setBigint(31,secods);
                                String formatBetween = DateUtil.formatBetween(ms, BetweenFormatter.Level.SECOND);
                                record.setString(32,formatBetween);
                            }

                            if(instanceSyncTaskMap.containsKey(instance.getNodeName())){
                                JSONObject jsonObject=instanceSyncTaskMap.get(instance.getNodeName());
                                record.setString(33,jsonObject.getString("totalBytes"));
                                record.setString(34,jsonObject.getString("totalRecords"));
                                record.setString(35,jsonObject.getString("errorRecords"));
                            }
                            instances.add(record);
                            mailRecords.add(record);
                            JobVarUtil.instanceMap.put(String.valueOf(instaceId),String.valueOf(instaceId));
                            // }
                        }
                    }
                    insertInstanceLog(bizDate, instances, "dw_instance_task_info_log");
                    instances=new ArrayList<>();
                } else {
                    System.out.println("Failed to list instances. Error code: " + response.getErrorCode() + ", Error message: " + response.getErrorMessage());
                }
            } catch (ClientException e) {
                e.printStackTrace();
            }
        }
        log.info("重复节点："+JSON.toJSONString(cfMap));
        return instances;
    }

    public static void getTaskLog(int page,Long projectId){
        File files = new File("cookie.txt");
        try {
            int totalPage=100;
            Date newDate = DateUtil.offset(new Date(), DateField.DAY_OF_MONTH, -1);
            String bizDate = DateUtil.format(newDate, "yyyy-MM-dd");
            String now=DateUtil.format(new Date(), "yyyy-MM-dd");
            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
            log.info("获取任务日志");
            while (page<=totalPage){
                List<Record> instances=new ArrayList<>();
                String cookies=readerMethod(files);
                String url="https://wkbench.res.cloud.cheryfs.cn/workbench/cwf/instance/list?projectId="+projectId+"&env=prod&tenantId=631873240849152&sortOrder=&sortField=&includeRelation=false&dagType=0&pageNum="+page+"&pageSize=100&forceUpdateId=6&expand=true&prgTypes=&searchText=&owner=&solId=&bizId=&taskTypes=0%2C2%2C3%2C5%2C6%2C7&startRunTimeFrom=&startRunTimeTo=&baseLineId=&bizdate="+bizDate+"%2000%3A00%3A00&beginBizDate=&endBizDate=&bizBeginHour="+now+"%2000%3A00%3A00&bizEndHour="+now+"%2023%3A59%3A00&connectionRegionId=&connectionType=&connections=&slowly=false&diResGroupIdentifier=&diSrcType=&diDstType=&diSrcDatasource=&diDstDatasource=&resgroupId=";
                String res= HttpRequest.get(url)
                        .cookie(cookies)
                        .execute().body();
                log.info("当前项目空间:"+ProjectEnums.getDesc(projectId)+" 当前页数:"+page);
                if(StringUtils.isNotEmpty(res)){
                    JSONObject jsonObject=JSONObject.parseObject(res);
                    if(jsonObject.getBoolean("success")){
                        JSONArray jsonArray=jsonObject.getJSONObject("data").getJSONArray("data");
                        int total=jsonObject.getJSONObject("data").getInteger("totalNum");
                        totalPage=(total+100-1)/100;
                        if(jsonArray!=null&&jsonArray.size()>0){
                            for (int i = 0; i < jsonArray.size(); i++) {
                                JSONObject jsdata=jsonArray.getJSONObject(i);
                                TaskInfoDto  taskInfoDto =JSONObject.toJavaObject(jsdata,TaskInfoDto.class);
                                Column[] columns1=new Column[36];
                                columns1[0]=(new Column("nodeId", OdpsType.valueOf("BIGINT")));
                                columns1[1]=(new Column("instanceId", OdpsType.valueOf("BIGINT")));
                                columns1[2]=(new Column("status", OdpsType.valueOf("STRING")));
                                columns1[3]=(new Column("bizdate", OdpsType.valueOf("STRING")));
                                columns1[4]=(new Column("cycTime", OdpsType.valueOf("TIMESTAMP")));
                                columns1[5]=(new Column("createTime", OdpsType.valueOf("TIMESTAMP")));
                                columns1[6]=(new Column("nodeName", OdpsType.valueOf("STRING")));
                                columns1[7]=(new Column("beginWaitTimeTime", OdpsType.valueOf("TIMESTAMP")));
                                columns1[8]=(new Column("beginWaitResTime", OdpsType.valueOf("TIMESTAMP")));
                                columns1[9]=(new Column("beginRunningTime", OdpsType.valueOf("TIMESTAMP")));
                                columns1[10]=(new Column("finishTime", OdpsType.valueOf("TIMESTAMP")));
                                columns1[11]=(new Column("priority", OdpsType.valueOf("BIGINT")));
                                columns1[12]=(new Column("errorMessage", OdpsType.valueOf("STRING")));
                                columns1[13]=(new Column("taskType", OdpsType.valueOf("STRING")));
                                columns1[14]=(new Column("taskRerunTime", OdpsType.valueOf("BIGINT")));
                                columns1[15]=(new Column("waitSchTimeDesc", OdpsType.valueOf("STRING")));
                                columns1[16]=(new Column("waitResTimeDesc", OdpsType.valueOf("STRING")));
                                columns1[17]=(new Column("runTimeDesc", OdpsType.valueOf("STRING")));
                                columns1[18]=(new Column("totalBytes", OdpsType.valueOf("STRING")));
                                columns1[19]=(new Column("totalNum", OdpsType.valueOf("STRING")));
                                columns1[20]=(new Column("error_records", OdpsType.valueOf("STRING")));
                                columns1[21]=(new Column("sys_name", OdpsType.valueOf("STRING")));
                                columns1[22]=(new Column("sys_ip", OdpsType.valueOf("STRING")));
                                columns1[23]=(new Column("data_sor_name", OdpsType.valueOf("STRING")));
                                columns1[24]=(new Column("cyc_type", OdpsType.valueOf("STRING")));
                                columns1[25]=(new Column("ower_name", OdpsType.valueOf("STRING")));
                                columns1[26]=(new Column("node_desc", OdpsType.valueOf("STRING")));
                                columns1[27]=(new Column("file_path", OdpsType.valueOf("STRING")));
                                columns1[28]=(new Column("sys_name_dest", OdpsType.valueOf("STRING")));
                                columns1[29]=(new Column("sys_ip_dest", OdpsType.valueOf("STRING")));
                                columns1[30]=(new Column("data_sor_name_dest", OdpsType.valueOf("STRING")));
                                columns1[31]=(new Column("item_space", OdpsType.valueOf("STRING")));
                                columns1[32]=(new Column("table_name", OdpsType.valueOf("STRING")));
                                columns1[33]=(new Column("table_comment", OdpsType.valueOf("STRING")));
                                columns1[34]=(new Column("table_name_dest", OdpsType.valueOf("STRING")));
                                columns1[35]=(new Column("table_dest_comment", OdpsType.valueOf("STRING")));
                                Record record=new ArrayRecord(columns1);
                                record.setBigint(0,taskInfoDto.getNodeId());
                                record.setBigint(1,taskInfoDto.getTaskId());
                                record.setString(2, DataWorksTaskInfoStatus.getDesc(taskInfoDto.getStatus()));
                                record.setString(3, sdf.format(new Date(taskInfoDto.getBizdate())));
                                record.setDatetime(4,new Timestamp(taskInfoDto.getCycTime()));
                                record.setDatetime(5,new Timestamp(taskInfoDto.getCreateTime()));
                                record.setString(6,taskInfoDto.getNodeName());
                                if(taskInfoDto.getBeginWaitTimeTime()!=null){
                                    record.setDatetime(7,new Timestamp(taskInfoDto.getBeginWaitTimeTime()));
                                }else {
                                    record.setDatetime(7,null);
                                }
                                if (taskInfoDto.getBeginWaitResTime()!=null){
                                    record.setDatetime(8,new Timestamp(taskInfoDto.getBeginWaitResTime()));
                                }else {
                                    record.setDatetime(8,null);
                                }
                                if(taskInfoDto.getBeginRunningTime()!=null){
                                    record.setDatetime(9,new Timestamp(taskInfoDto.getBeginRunningTime()));
                                }else {
                                    record.setDatetime(9,null);
                                }
                                if(taskInfoDto.getFinishTime()!=null){
                                    record.setDatetime(10,new Timestamp(taskInfoDto.getFinishTime()));
                                }else {
                                    record.setDatetime(10,null);
                                }
                                record.setBigint(11,taskInfoDto.getPriority().longValue());
                                record.setString(12,"");
                                if(DataWorksTaskInfoStatus.getDesc(taskInfoDto.getStatus()).equals("FAILURE")){
                                    log.info("nodeId:"+taskInfoDto.getNodeId());
                                    String log=getInstanceLog(taskInfoDto.getTaskId());
                                    record.setString(12,log);
                                }
                                record.setString(13,taskInfoDto.getPrgTypeName());
                                if(taskInfoDto.getTaskRerunTime()!=null){
                                    record.setBigint(14,taskInfoDto.getTaskRerunTime().longValue());
                                }else {
                                    record.setBigint(14,0L);
                                }
                                if(taskInfoDto.getBeginWaitTimeTime()!=null&&taskInfoDto.getBeginWaitResTime()!=null){
                                    Long secods = DateUtil.between(new Date(taskInfoDto.getBeginWaitTimeTime()), new Date(taskInfoDto.getBeginWaitResTime()), DateUnit.SECOND);
                                    Long ms = DateUtil.between(new Date(taskInfoDto.getBeginWaitTimeTime()), new Date(taskInfoDto.getBeginWaitResTime()), DateUnit.MS);
                                    String formatBetween = DateUtil.formatBetween(ms, BetweenFormatter.Level.SECOND);
                                    record.setString(15,formatBetween);
                                }
                                if(taskInfoDto.getBeginWaitResTime()!=null&&taskInfoDto.getBeginRunningTime()!=null){
                                    Long secods = DateUtil.between(new Date(taskInfoDto.getBeginWaitResTime()), new Date(taskInfoDto.getBeginRunningTime()), DateUnit.SECOND);
                                    Long ms = DateUtil.between(new Date(taskInfoDto.getBeginWaitResTime()), new Date(taskInfoDto.getBeginRunningTime()), DateUnit.MS);
                                    //  record.setBigint(29,secods);
                                    String formatBetween = DateUtil.formatBetween(ms, BetweenFormatter.Level.SECOND);
                                    record.setString(16,formatBetween);
                                }
                                if(taskInfoDto.getBeginRunningTime()!=null&&taskInfoDto.getFinishTime()!=null){
                                    Long secods = DateUtil.between(new Date(taskInfoDto.getBeginRunningTime()), new Date(taskInfoDto.getFinishTime()), DateUnit.SECOND);
                                    Long ms = DateUtil.between(new Date(taskInfoDto.getBeginRunningTime()), new Date(taskInfoDto.getFinishTime()), DateUnit.MS);
                                    //    record.setBigint(31,secods);
                                    String formatBetween = DateUtil.formatBetween(ms, BetweenFormatter.Level.SECOND);
                                    record.setString(17,formatBetween);
                                }
                                //   record.setString(35,"0");
//                                if(instanceSyncTaskMap.containsKey(taskInfoDto.getNodeName())){
//                                    JSONObject syncTask=instanceSyncTaskMap.get(taskInfoDto.getNodeName());
//                                    record.setString(18,syncTask.getString("totalBytes"));
//                                    record.setString(19,syncTask.getString("totalRecords"));
//                                    record.setString(20,syncTask.getString("errorRecords"));
//                                }
                                record.setString(21,"");
                                record.setString(22,"");
                                record.setString(23,"");
                                record.setString(32,"");
                                record.setString(33,"");
                                record.setString(34,"");
                                record.setString(35,"");
                                if(taskInfoDto.getPrgTypeName().equals("离线同步")){
                                    DataTaskInfoDto dataTaskInfoDto=getDataSourceInfoByNodeCode(taskInfoDto.getNodeId().toString());
                                    String log=getInstanceLog(taskInfoDto.getTaskId());
                                    SyncDataStats syncDataStats=LogParserUtil.parseLog(log);
                                    record.setString(18,"");
                                    record.setString(19,"");
                                    record.setString(20,"");
                                    if(syncDataStats!=null){
                                        record.setString(18,syncDataStats.getTotalSize());
                                        record.setString(19,String.valueOf(syncDataStats.getTotalRecords()));
                                        record.setString(20,String.valueOf(syncDataStats.getDirtyRecords()));
                                    }
                                    if(dataTaskInfoDto!=null&&StringUtils.isNotEmpty(dataTaskInfoDto.getSysName())){
                                        record.setString(21,dataTaskInfoDto.getSysName());
                                        record.setString(22,dataTaskInfoDto.getSysIp());
                                        record.setString(23,dataTaskInfoDto.getDataSorName());
                                        record.setString(28,dataTaskInfoDto.getSysNameDest());
                                        record.setString(29,dataTaskInfoDto.getSysIpDest());
                                        record.setString(30,dataTaskInfoDto.getDataSorNameDest());
                                        record.setString(32,dataTaskInfoDto.getTableName());
                                        record.setString(33,dataTaskInfoDto.getTableNameComment());
                                        record.setString(34,dataTaskInfoDto.getTableNameDest());
                                        record.setString(35,dataTaskInfoDto.getTableNameDestComment());
                                    }
                                }
                                if(taskInfoDto.getPrgTypeName().equals("ODPS SQL")||taskInfoDto.getPrgTypeName().equals("ODPS Script")){
                                    String nodeId=String.valueOf(taskInfoDto.getNodeId());
                                    log.info(nodeId);
                                    String tableName="";
                                    if(nodeId.equals("22982")){
                                        tableName="dim_public_cd_tbl";
                                    }else{
                                        tableName=exactTableName(getNodeCode(nodeId));
                                    }
                                    if(StringUtils.isNotEmpty(tableName)){
                                        String comment="";
                                        tableName=tableName.toLowerCase();
                                        String engin=ProjectEnums.getEngineByCode(projectId);
                                        String[] envs=ProjectEnums.getEnvByCode(projectId);
                                        for(String env:envs){
                                            tableName=tableName.replace(env+".","");
                                        }
                                        GetMetaTableBasicInfoResponse.Data data=getMetaTableBasicInfo(engin+"."+tableName);
                                        if(data!=null){
                                            comment=data.getCaption();
                                        }
                                        record.setString(34,tableName);
                                        record.setString(35,comment);
                                    }
                                    if(String.valueOf(projectId).equals("10022")){
                                        String nodeName=taskInfoDto.getNodeName();
                                        String status=DataWorksTaskInfoStatus.getDesc(taskInfoDto.getStatus());
                                        String monitorScript= config.getStr("monitorScript","");
                                        if(StringUtils.isNotEmpty(monitorScript)){
                                            String[] monitor=monitorScript.split(",");
                                            for (String str:monitor){
                                                if(StringUtils.isNotEmpty(str)){
                                                    if(nodeName.equals(str)&&!status.equals("SUCCESS")){
                                                        phoneWarning("重点监控表跑批存在问题请前往运维中心查看",30);
                                                        break;
                                                    }
                                                }
                                            }
                                        }
                                    }
                                }
                                record.setString(24,taskInfoDto.getCycTypeName());
                                record.setString(25,taskInfoDto.getOwnerName());
                                String desc="";
                                GetNodeResponse.Data data=getNodeByNodeId(taskInfoDto.getNodeId());
                                if(data!=null){
                                    if(!data.getDescription().contains("cloned from")){
                                        desc=data.getDescription();
                                    }
                                }
                                record.setString(26,desc);
                                ListFilesResponse.Data.File file=getFile(taskInfoDto.getNodeId().toString(),projectId);
                                String filePath="";
                                if(file!=null){
                                    String folderId=file.getFileFolderId();
                                    GetFolderResponse.Data folder=getFolders(folderId,projectId);
                                    if(folder!=null){
                                        filePath=folder.getFolderPath().replace("folderDi","数据集成")
                                                .replace("folderMaxCompute","MaxCompute/数据开发")
                                                .replace("folderGeneral","通用")
                                        ;
                                    }
                                }
                                record.setString(27,filePath);
                                record.setString(31, ProjectEnums.getDesc(projectId));
                                String  instanceId=String.valueOf(taskInfoDto.getTaskId());
                                if(errorTask.containsKey(instanceId)){
                                    String status=DataWorksTaskInfoStatus.getDesc(taskInfoDto.getStatus());
                                    if(status.equals("SUCCESS")){
                                        log.info("发现重跑成功实列:"+JSON.toJSONString(errorTask.get(instanceId)));
                                        errorTask.remove(instanceId);
                                    }
                                }
                                instances.add(record);
                                //    mailRecords.add(record);
                                // JobVarUtil.instanceMap.put(String.valueOf(instaceId),String.valueOf(instaceId));
                            }
                        }
                    }
                    page++;
                   insertInstanceLog(bizDate.replace("-",""),instances, "dataops_run_batch_monitor");
                }else{
                    login();
                    continue;
                }
            }

        }catch (Exception e){
            e.printStackTrace();
            if(retryCount<=3){
                retryCount++;
                getTaskLog(page,projectId);
                try {
                    Thread.sleep(60000);
                } catch (InterruptedException ex) {
                    throw new RuntimeException(ex);
                }
            }
        }
    }

    public static void insertTaskError(List<InstanceEntityDto> instanceEntityDtos,boolean isExtRule){
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        Date newDate = DateUtil.offset(new Date(), DateField.DAY_OF_MONTH, -1);
        String bizDate = DateUtil.format(newDate, "yyyyMMdd");
        List<Record> records=new ArrayList<>();
        if(datasources.size()==0){
            getDatasource();
        }
        for (InstanceEntityDto taskInfoDto:instanceEntityDtos){
            Column[] columns1=new Column[20];
            columns1[0]=(new Column("nodeId", OdpsType.valueOf("BIGINT")));
            columns1[1]=(new Column("instanceId", OdpsType.valueOf("BIGINT")));
            columns1[2]=(new Column("bizdate", OdpsType.valueOf("STRING")));
            columns1[3]=(new Column("cycTime", OdpsType.valueOf("TIMESTAMP")));
            columns1[4]=(new Column("createTime", OdpsType.valueOf("TIMESTAMP")));
            columns1[5]=(new Column("nodeName", OdpsType.valueOf("STRING")));
            columns1[6]=(new Column("beginWaitTimeTime", OdpsType.valueOf("TIMESTAMP")));
            columns1[7]=(new Column("beginWaitResTime", OdpsType.valueOf("TIMESTAMP")));
            columns1[8]=(new Column("beginRunningTime", OdpsType.valueOf("TIMESTAMP")));
            columns1[9]=(new Column("finishTime", OdpsType.valueOf("TIMESTAMP")));
            columns1[10]=(new Column("errorMessage", OdpsType.valueOf("STRING")));
            columns1[11]=(new Column("errorDesc", OdpsType.valueOf("STRING")));
            columns1[12]=(new Column("nodeType", OdpsType.valueOf("STRING")));
            columns1[13]=(new Column("taskRerunTime", OdpsType.valueOf("BIGINT")));
            columns1[14]=(new Column("sys_name", OdpsType.valueOf("STRING")));
            columns1[15]=(new Column("sys_ip", OdpsType.valueOf("STRING")));
            columns1[16]=(new Column("data_sor_name", OdpsType.valueOf("STRING")));
            columns1[17]=(new Column("node_desc", OdpsType.valueOf("STRING")));
            columns1[18]=(new Column("owner_name", OdpsType.valueOf("STRING")));
            columns1[19]=(new Column("etl_time", OdpsType.valueOf("STRING")));
            Record record=new ArrayRecord(columns1);
            record.setBigint(0,Long.valueOf(taskInfoDto.getNodeId()));
            record.setBigint(1,taskInfoDto.getTaskId());
            record.setString(2, sdf.format(new Date(taskInfoDto.getBizdate())));
            record.setDatetime(3,new Timestamp(taskInfoDto.getCycTime()));
            record.setDatetime(4,new Timestamp(taskInfoDto.getCreateTime()));
            record.setString(5,taskInfoDto.getNodeName());
            if(Long.valueOf(taskInfoDto.getBeginWaitTimeTime())!=null){
                record.setDatetime(6,new Timestamp(taskInfoDto.getBeginWaitTimeTime()));
            }else {
                record.setDatetime(6,null);
            }
            if (Long.valueOf(taskInfoDto.getBeginWaitResTime())!=null){
                record.setDatetime(7,new Timestamp(taskInfoDto.getBeginWaitResTime()));
            }else {
                record.setDatetime(7,null);
            }
            if(Long.valueOf(taskInfoDto.getBeginRunningTime())!=null){
                record.setDatetime(8,new Timestamp(taskInfoDto.getBeginRunningTime()));
            }else {
                record.setDatetime(8,null);
            }
            if((taskInfoDto.getFinshTime())!=null){
                record.setDatetime(9,new Timestamp(taskInfoDto.getFinshTime()));
            }else {
                record.setDatetime(9,null);
            }
            String errorLog=getInstanceLog(taskInfoDto.getTaskId());
            String logDesc=LogParserUtil.getlogErrorInfo(errorLog);
            record.setString(10,errorLog);
            record.setString(11,logDesc);
            record.setString(12,taskInfoDto.getPrgTypeName());
            if(Long.valueOf(taskInfoDto.getTaskRerunTime())!=null){
                record.setBigint(13,taskInfoDto.getTaskRerunTime());
            }else {
                record.setBigint(13,0L);
            }
            if(taskInfoDto.getPrgTypeName().equals("离线同步")){
                TableDesDto tableDesDto=getDataSourceByNodeCode(String.valueOf(taskInfoDto.getNodeId()));
                if(tableDesDto!=null&&StringUtils.isNotEmpty(tableDesDto.getSysName())){
                    record.setString(14,tableDesDto.getSysName());
                    record.setString(15,tableDesDto.getSysIp());
                    record.setString(16,tableDesDto.getDataSorName());
                }
            }
            String desc="";
            GetNodeResponse.Data data=getNodeByNodeId(Long.valueOf(taskInfoDto.getNodeId()));
            if(data!=null){
                if(!data.getDescription().contains("cloned from")){
                    desc=data.getDescription();
                }
            }
            record.setString(17,desc);
            record.setString(18,taskInfoDto.getOwnerName());
            record.setString(19,sdf.format(new Date()));
            String nodeName=taskInfoDto.getNodeName();
            String warnRule= config.getStr("warnRule","");
            log.info("warnRule:"+warnRule);
            boolean isExtSuccess=false;
            boolean isAllExtSuccess=false;
            if(isExtRule){
                records.add(record);
            }else{
                if(warnRule.equals("*")){
                    isAllExtSuccess=true;
                }else{
                    if(StringUtils.isNotEmpty(warnRule)&&StringUtils.isNotEmpty(logDesc)){
                        for (String warn:warnRule.split("\\|")){
                            if(StringUtils.isNotEmpty(warn)&&logDesc.contains(warn.trim())){
                                isExtSuccess=true;
                                log.info("错误描述:"+logDesc);
                                log.info("匹配到的规则:"+warn);
                                break;
                            }
                        }
                    }
                }
                if(isAllExtSuccess){
                    String instanceId=String.valueOf(taskInfoDto.getTaskId());
                    log.info("进入配置*规则");
                    collectErrorTask(instanceId,record);
                }else{
                    if(StringUtils.isNotEmpty(logDesc)&&isExtSuccess){
                        String instanceId=String.valueOf(taskInfoDto.getTaskId());
                        log.info("进入配置warRule规则");
                        collectErrorTask(instanceId,record);
                    }else{
                        if(!nodeName.contains("dws_bsport_area_revenue_income_month_di")){
                            records.add(record);
                        }
                    }
                }
            }
        }
        if(records!=null&&records.size()>0){
            insertInstanceLog(bizDate,records,"dataops_run_batch_error_info");
        }
    }


    public static void collectErrorTask(String instanceId,Record record){
        if(!errorTask.containsKey(instanceId)){
            List<Record> errors=new ArrayList<>();
            errors.add(record);
            errorTask.put(instanceId,errors);
        }else{
            List<Record> errors=errorTask.get(instanceId);
            errors.add(record);
            errorTask.put(instanceId,errors);
        }
    }

    public static TableDesDto getDataSourceByNodeCode(String nodeId){
        GetNodeCodeRequest request = new GetNodeCodeRequest();
        request.setNodeId(Long.valueOf(nodeId));
        request.setProjectEnv("PROD");
        String res="";
        TableDesDto tableDes=new TableDesDto();
        try {
            GetNodeCodeResponse response = client.getAcsResponse(request);
            if (response.getSuccess()) {
                res=response.getData();
                JSONArray jsonObject= JSONObject.parseObject(res).getJSONArray("steps");
                String tableName="";
                String source="";
                JSONObject reader=getObjectByKey(jsonObject,"name","Reader");
                JSONObject writer=getObjectByKey(jsonObject,"name","Writer");
                JSONObject readerJs=reader.getJSONObject("parameter");
                JSONObject writerJs=writer.getJSONObject("parameter");
                String stepType=reader.getString("stepType");
                String writeStepType=writer.getString("stepType");
                //集成
                tableDes.setNodeId(nodeId);
                //抽数
                if(!stepType.equals("odps")){
                    if(stepType.equals("mysql")){
                        source=readerJs.getJSONArray("connection").getJSONObject(0).getString("datasource");
                        tableName=readerJs.getJSONArray("connection").getJSONObject(0).getJSONArray("table").getString(0).toLowerCase();
                    }else{
                        source=readerJs.getString("datasource");
                        tableName=readerJs.getString("table").toLowerCase();
                    }
                    tableDes.setTableName(tableName);
                    String desName=writerJs.getString("table");
                    if(datasources.containsKey(source)){
                        ListDataSourcesResponse.Data.DataSourcesItem item=datasources.get(source);
                        tableDes.setSysName(item.getDescription());
                        tableDes.setSysIp(extractIP(item.getDescription()));
                        tableDes.setSourceTableName(tableName);
                        tableDes.setDesTableName(desName);
                        tableDes.setNodeId(nodeId);
                        tableDes.setDataSorName(source);
//                        if(!inMap.containsKey(tableName)){
//                            tableDes.setSourceTableName(tableName);
//                            tableDes.setDesTableName(desName);
//                            inMap.put(tableName,tableDes);
//                        }else{
//                            TableDes tableDes1=inMap.get(tableName);
//                            tableDes1.setNodeId(tableDes1.getNodeId()+","+nodeId);
//                            tableDes1.setSourceTableName(tableName);
//                            tableDes1.setDesTableName(tableDes1.getDesTableName()+","+desName);
//                            inMap.put(tableName,tableDes1);
//                        }
                    }
                }else{
                    //推送
                    if(writeStepType.equals("ftp")){
                        source=writerJs.getString("datasource");
                        tableName=readerJs.getString("table").toLowerCase();
                    }else{
                        source=writerJs.getString("datasource");
                        tableName=writerJs.getString("table").toLowerCase();
                    }
                    String soureName=readerJs.getString("table");
                    //排除dataworks到dataworks直抽
                    if(!source.equals("odps_first")){
                        tableDes.setTableName(tableName);
                        if(datasources.containsKey(source)){
                            ListDataSourcesResponse.Data.DataSourcesItem item=datasources.get(source);
                            tableDes.setSysName(item.getDescription());
                            tableDes.setSysIp(extractIP(item.getDescription()));
                            tableDes.setDesTableName(tableName);
                            tableDes.setSourceTableName(soureName);
                            tableDes.setDataSorName(source);
                            tableDes.setNodeId(nodeId);
//                            if(!pushMap.containsKey(tableName)){
//                                tableDes.setDesTableName(tableName);
//                                tableDes.setSourceTableName(soureName);
//                                pushMap.put(tableName,tableDes);
//                            }else{
//                                TableDes tableDes1=pushMap.get(tableName);
//                                tableDes1.setNodeId(tableDes1.getNodeId()+","+nodeId);
//                                tableDes1.setDesTableName(tableDes1.getDesTableName()+","+tableName);
//                                tableDes1.setSourceTableName(tableDes1.getSourceTableName()+","+soureName);
//                                pushMap.put(tableName,tableDes1);
//                            }
                        }
                    }
                }
            } else {
                System.out.println("Failed to get node code. Error code: " + response.getErrorCode() + ", Error message: " + response.getErrorMessage());
            }
        }catch (Exception e) {
            String mess=e.getMessage();
            if(mess.contains("对应的节点信息不存在")){
                System.out.println("节点不存在");
            }else{
                throw new RuntimeException(e);
            }
        }
        return tableDes;
    }

    public static DataTaskInfoDto getDataSourceInfoByNodeCode(String nodeId){
        GetNodeCodeRequest request = new GetNodeCodeRequest();
        request.setNodeId(Long.valueOf(nodeId));
        request.setProjectEnv("PROD");
        String res="";
        DataTaskInfoDto dataTaskInfoDto=new DataTaskInfoDto();
        try {
            GetNodeCodeResponse response = client.getAcsResponse(request);
            if (response.getSuccess()) {
                res=response.getData();
                JSONArray jsonObject= JSONObject.parseObject(res).getJSONArray("steps");
                String source="";
                JSONObject reader=getObjectByKey(jsonObject,"name","Reader");
                JSONObject writer=getObjectByKey(jsonObject,"name","Writer");
                JSONObject readerJs=reader.getJSONObject("parameter");
                JSONObject writerJs=writer.getJSONObject("parameter");
                String stepType=reader.getString("stepType");
                String writeStepType=writer.getString("stepType");
                //抽数
                if(stepType.equals("odps")){
                    dataTaskInfoDto.setSysIp("odps");
                    dataTaskInfoDto.setSysName("odps");
                    dataTaskInfoDto.setDataSorName(readerJs.getString("datasource"));
                    String tableName=readerJs.getString("table");
                    String tableNameComment=readerJs.getString("tableComment");
                    dataTaskInfoDto.setTableName(tableName);
                    dataTaskInfoDto.setTableNameComment(StringUtils.isNotEmpty(tableNameComment)?tableNameComment.replace("null",""):null);
                }else{
                    if(stepType.equals("mysql")){
                        source=readerJs.getJSONArray("connection").getJSONObject(0).getString("datasource");
                        String tableName=readerJs.getJSONArray("connection").getJSONObject(0).getJSONArray("table").getString(0);
                        String tableNameComment=readerJs.getString("tableComment");
                        dataTaskInfoDto.setTableName(tableName);
                        dataTaskInfoDto.setTableNameComment(StringUtils.isNotEmpty(tableNameComment)?tableNameComment.replace("null",""):null);
                    }else{
                        source=readerJs.getString("datasource");
                        String tableName=readerJs.getString("table");
                        dataTaskInfoDto.setTableName(tableName);
                    }
                    if(datasources.containsKey(source)){
                        ListDataSourcesResponse.Data.DataSourcesItem item=datasources.get(source);
                        dataTaskInfoDto.setSysName(item.getDescription());
                        dataTaskInfoDto.setSysIp(extractIP(item.getDescription()));
                        dataTaskInfoDto.setDataSorName(source);
                    }
                }
                if(writeStepType.equals("odps")){
                    dataTaskInfoDto.setSysIpDest("odps");
                    dataTaskInfoDto.setSysNameDest("odps");
                    dataTaskInfoDto.setDataSorNameDest(writerJs.getString("datasource"));
                    String tableNameDest=writerJs.getString("table");
                    String tableNameCommentDest=writerJs.getString("tableComment");
                    dataTaskInfoDto.setTableNameDest(tableNameDest);
                    dataTaskInfoDto.setTableNameDestComment(StringUtils.isNotEmpty(tableNameCommentDest)?tableNameCommentDest.replace("null",""):null);
                }else{
                    if(writeStepType.equals("ftp")){
                        source=writerJs.getString("datasource");
                    }else{
                        source=writerJs.getString("datasource");
                        String tableNameDest=writerJs.getString("table");
                        dataTaskInfoDto.setTableNameDest(tableNameDest);
                    }
                    if(datasources.containsKey(source)){
                        ListDataSourcesResponse.Data.DataSourcesItem item=datasources.get(source);
                        dataTaskInfoDto.setSysNameDest(item.getDescription());
                        dataTaskInfoDto.setSysIpDest(extractIP(item.getDescription()));
                        dataTaskInfoDto.setDataSorNameDest(source);
                    }
                }
//                if(!stepType.equals("odps")){
//                    if(stepType.equals("mysql")){
//                        source=readerJs.getJSONArray("connection").getJSONObject(0).getString("datasource");
//                        tableName=readerJs.getJSONArray("connection").getJSONObject(0).getJSONArray("table").getString(0).toLowerCase();
//                    }else{
//                        source=readerJs.getString("datasource");
//                        tableName=readerJs.getString("table").toLowerCase();
//                    }
//                    tableDes.setTableName(tableName);
//                    String desName=writerJs.getString("table");
//                    if(datasources.containsKey(source)){
//                        ListDataSourcesResponse.Data.DataSourcesItem item=datasources.get(source);
//                        tableDes.setSysName(item.getDescription());
//                        tableDes.setSysIp(extractIP(item.getDescription()));
//                        tableDes.setSourceTableName(tableName);
//                        tableDes.setDesTableName(desName);
//                        tableDes.setNodeId(nodeId);
//                        tableDes.setDataSorName(source);
//                    }
//                }else{
//                    //推送
//                    if(writeStepType.equals("ftp")){
//                        source=writerJs.getString("datasource");
//                        tableName=readerJs.getString("table").toLowerCase();
//                    }else{
//                        source=writerJs.getString("datasource");
//                        tableName=writerJs.getString("table").toLowerCase();
//                    }
//                    String soureName=readerJs.getString("table");
//                    //排除dataworks到dataworks直抽
//                    if(!source.equals("odps_first")){
//                        tableDes.setTableName(tableName);
//                        if(datasources.containsKey(source)){
//                            ListDataSourcesResponse.Data.DataSourcesItem item=datasources.get(source);
//                            tableDes.setSysName(item.getDescription());
//                            tableDes.setSysIp(extractIP(item.getDescription()));
//                            tableDes.setDesTableName(tableName);
//                            tableDes.setSourceTableName(soureName);
//                            tableDes.setDataSorName(source);
//                            tableDes.setNodeId(nodeId);
//                        }
//                    }
//                }
            } else {
                System.out.println("Failed to get node code. Error code: " + response.getErrorCode() + ", Error message: " + response.getErrorMessage());
            }
        }catch (Exception e) {
            String mess=e.getMessage();
            if(mess.contains("对应的节点信息不存在")){
                System.out.println("节点不存在");
            }else{
                throw new RuntimeException(e);
            }
        }
        return dataTaskInfoDto;
    }

    public static DataTaskInfoDto getDataSourceInfoByNodeCode(String nodeId,Long project){
        GetNodeCodeRequest request = new GetNodeCodeRequest();
        request.setNodeId(Long.valueOf(nodeId));
        request.setProjectEnv("PROD");
        String res="";
        DataTaskInfoDto dataTaskInfoDto=new DataTaskInfoDto();
        try {
            GetNodeCodeResponse response = client.getAcsResponse(request);
            if (response.getSuccess()) {
                res=response.getData();
                JSONArray jsonObject= JSONObject.parseObject(res).getJSONArray("steps");
                String source="";
                JSONObject reader=getObjectByKey(jsonObject,"name","Reader");
                JSONObject writer=getObjectByKey(jsonObject,"name","Writer");
                JSONObject readerJs=reader.getJSONObject("parameter");
                JSONObject writerJs=writer.getJSONObject("parameter");
                String stepType=reader.getString("stepType");
                String writeStepType=writer.getString("stepType");
                //抽数
                if(stepType.equals("odps")){
                    dataTaskInfoDto.setSysIp("odps");
                    dataTaskInfoDto.setSysName("odps");
                    dataTaskInfoDto.setDataSorName(readerJs.getString("datasource"));
                }else{
                    if(stepType.equals("mysql")){
                        source=readerJs.getJSONArray("connection").getJSONObject(0).getString("datasource");
                    }else{
                        source=readerJs.getString("datasource");
                    }
                    if(datasources.containsKey(source)){
                        ListDataSourcesResponse.Data.DataSourcesItem item=datasources.get(source);
                        dataTaskInfoDto.setSysName(item.getDescription());
                        dataTaskInfoDto.setSysIp(extractIP(item.getDescription()));
                        dataTaskInfoDto.setDataSorName(source);
                    }
                }
                if(writeStepType.equals("odps")){
                    dataTaskInfoDto.setSysIpDest("odps");
                    dataTaskInfoDto.setSysNameDest("odps");
                    dataTaskInfoDto.setDataSorNameDest(writerJs.getString("datasource"));
                }else{
                    if(writeStepType.equals("ftp")){
                        source=writerJs.getString("datasource");
                    }else{
                        source=writerJs.getString("datasource");
                    }
                    if(datasources.containsKey(source)){
                        ListDataSourcesResponse.Data.DataSourcesItem item=datasources.get(source);
                        dataTaskInfoDto.setSysNameDest(item.getDescription());
                        dataTaskInfoDto.setSysIpDest(extractIP(item.getDescription()));
                        dataTaskInfoDto.setDataSorNameDest(source);
                    }
                }
//                if(!stepType.equals("odps")){
//                    if(stepType.equals("mysql")){
//                        source=readerJs.getJSONArray("connection").getJSONObject(0).getString("datasource");
//                        tableName=readerJs.getJSONArray("connection").getJSONObject(0).getJSONArray("table").getString(0).toLowerCase();
//                    }else{
//                        source=readerJs.getString("datasource");
//                        tableName=readerJs.getString("table").toLowerCase();
//                    }
//                    tableDes.setTableName(tableName);
//                    String desName=writerJs.getString("table");
//                    if(datasources.containsKey(source)){
//                        ListDataSourcesResponse.Data.DataSourcesItem item=datasources.get(source);
//                        tableDes.setSysName(item.getDescription());
//                        tableDes.setSysIp(extractIP(item.getDescription()));
//                        tableDes.setSourceTableName(tableName);
//                        tableDes.setDesTableName(desName);
//                        tableDes.setNodeId(nodeId);
//                        tableDes.setDataSorName(source);
//                    }
//                }else{
//                    //推送
//                    if(writeStepType.equals("ftp")){
//                        source=writerJs.getString("datasource");
//                        tableName=readerJs.getString("table").toLowerCase();
//                    }else{
//                        source=writerJs.getString("datasource");
//                        tableName=writerJs.getString("table").toLowerCase();
//                    }
//                    String soureName=readerJs.getString("table");
//                    //排除dataworks到dataworks直抽
//                    if(!source.equals("odps_first")){
//                        tableDes.setTableName(tableName);
//                        if(datasources.containsKey(source)){
//                            ListDataSourcesResponse.Data.DataSourcesItem item=datasources.get(source);
//                            tableDes.setSysName(item.getDescription());
//                            tableDes.setSysIp(extractIP(item.getDescription()));
//                            tableDes.setDesTableName(tableName);
//                            tableDes.setSourceTableName(soureName);
//                            tableDes.setDataSorName(source);
//                            tableDes.setNodeId(nodeId);
//                        }
//                    }
//                }
            } else {
                System.out.println("Failed to get node code. Error code: " + response.getErrorCode() + ", Error message: " + response.getErrorMessage());
            }
        }catch (Exception e) {
            String mess=e.getMessage();
            if(mess.contains("对应的节点信息不存在")){
                System.out.println("节点不存在");
            }else{
                throw new RuntimeException(e);
            }
        }
        return dataTaskInfoDto;
    }

    public static JSONObject getObjectByKey(JSONArray jsonArray, String key, String value) {
        for (int i = 0; i < jsonArray.size(); i++) {
            JSONObject jsonObject = jsonArray.getJSONObject(i);
            if (value.equals(jsonObject.getString(key))) {
                return jsonObject;
            }
        }
        return null;
    }

    public  static Long gpsMonitor(String bizDate,String tableName,String project){
       Long total=0L;
        Account account = new AliyunAccount("feupFUuHYOxrohjn", "MUFrk9HsB94LXzyDZZA0lhPvvoa1J0");
        Odps odps = new Odps(account);
        String odpsUrl = "http://service.cn-wuhu-qrhy-d01.odps.ops.cloud.cheryfs.cn/api";
        odps.setEndpoint(odpsUrl);
        odps.setDefaultProject(project);
        TableTunnel tunnel = new TableTunnel(odps);
        PartitionSpec partitionSpec=new PartitionSpec();
        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd");
        LocalDate date = LocalDate.parse(bizDate, formatter);
        int year = date.getYear();
        int month = date.getMonthValue();
        int day = date.getDayOfMonth();
        partitionSpec.set("year",String.valueOf(year));
        partitionSpec.set("month", String.format("%02d", month));
        partitionSpec.set("day", String.format("%02d", day));
        Table table = odps.tables().get(tableName);//获取当前表
        try {

           boolean partionExists= table.hasPartition(partitionSpec);
           log.info(project+" "+tableName+" "+partionExists);
           if(!partionExists){
               phoneWarning("数据中台实时同步数据出现问题");
               return total;
           }
           long count= tunnel.createDownloadSession(project,tableName, partitionSpec,true).getRecordCount();
           log.info("数据量:"+count);
           if(count<1000000000L){
               phoneWarning("数据中台实时同步数据量过小");
           }
           total=Long.valueOf(count);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return total;
    }

    public static void runManualDagNodes(String bizDate,String flowName,String projectName,String nodeIds,Long projectId){

        try {
            RunManualDagNodesRequest request = new RunManualDagNodesRequest();
            request.setBizDate(bizDate);
            request.setProjectEnv("PROD");
            request.setFlowName(flowName);
            request.setIncludeNodeIds(nodeIds);
            request.setProjectName(projectName);
            request.setProjectId(projectId);
            RunManualDagNodesResponse response = client.getAcsResponse(request);
            log.info(JSON.toJSONString(response));
        } catch (Exception e) {
            e.printStackTrace();
        }
    }


    public static String submitTable(String tableName){
        File files = new File("cookie.txt");
        String res="";
        try {
            String cookies=readerMethod(files);
            String url="https://ide.res.cloud.cheryfs.cn/rest/table/detail?projectId=10022&tenantId=631873240849152&envType=0&appGuid=odps.DatawareHouse_Dev&srcType=0&tableName="+tableName+"&tableGuid=odps.datawarehouse_dev."+tableName+"&exceptionIfNotExists=false";
            String result= HttpRequest.get(url)
                    .cookie(cookies)
                    .execute().body();
            if(StringUtils.isNotEmpty(result)){
                JSONObject jsonObject= JSON.parseObject(result);
                if(jsonObject.getBoolean("success")){
                    JSONObject data=jsonObject.getJSONObject("data");
                  TableDetailDto tableDetailDto=JSON.toJavaObject(data,TableDetailDto.class);
                  SubmitTableDto submitTableDto=JSON.toJavaObject(data,SubmitTableDto.class);
                  submitTableDto.setAppGuid("odps.datawarehouse_dev");
                  submitTableDto.setPhysicsLevelId(0);
                  if(StringUtils.isEmpty(submitTableDto.getTableDesc())){
                      submitTableDto.setTableDesc(".....");
                  }
                    if(StringUtils.isEmpty(submitTableDto.getTableNameCn())){
                        if(StringUtils.isNotEmpty(submitTableDto.getTableDesc())){
                            submitTableDto.setTableNameCn(submitTableDto.getTableDesc());
                        }else{
                            submitTableDto.setTableNameCn(".....");
                        }
                    }
                    submitTableDto.setSrcType(0);
                    submitTableDto.setEnvType(1);
                    submitTableDto.setUserId("5246017503763386908");
                    submitTableDto.setTableGuid(null);
                    String detail=JSON.toJSONString(tableDetailDto);
                    res=JSON.toJSONString(submitTableDto);
                  String preCheckUrl="https://ide.res.cloud.cheryfs.cn/rest/ddl/ddl";
                    String token=getxcsrfToken();
                    String checkResult= HttpRequest.post(preCheckUrl)
                            .header(Header.ACCEPT, "application/json")
                            .header(Header.COOKIE, cookies+"; "+"csrf-token="+token)
                            .header("X-Csrf-Token",token)
                            .body(detail)
                            .execute().body();
                    log.info("检查结果:"+checkResult);
                    if(StringUtils.isNotEmpty(checkResult)){
                        JSONObject checkJson=JSON.parseObject(checkResult);
                        if(checkJson.getBoolean("success")){
                            String submitUrl="https://ide.res.cloud.cheryfs.cn/rest/table/1?force=false";
                            String submitResult= HttpRequest.put(submitUrl)
                                    .header(Header.ACCEPT, "application/json")
                                    .header(Header.COOKIE, cookies+"; "+"csrf-token="+token)
                                    .header("X-Csrf-Token",token)
                                    .body(res)
                                    .execute().body();
                            log.info("提交结果:"+submitResult);
                        }
                    }

                }
            }else{
                log.info("查询结果为空 尝试重新登陆");
                login();
                return submitTable(tableName);
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return res;
    }

    public static List<String>  parseLogForTableNotFound(String logContent) {
        if(StringUtils.isEmpty(logContent)){
            return null;
        }
        List<String> tableName=new ArrayList<>();
        Pattern pattern = Pattern.compile("FAILED: ODPS-0130131:\\[\\d+,\\d+\\] Table not found - table (\\S+) cannot be resolved");
        Matcher matcher = pattern.matcher(logContent);
        StringBuffer sb=new StringBuffer();
        while (matcher.find()) {
            String name=matcher.group(1);
             tableName.add(name);
           log.info("找到表不存在错误，表名: " + name);
        }
        return tableName;
    }

    private static void getTableFullInfo(String table){
        GetMetaTableFullInfoRequest request = new GetMetaTableFullInfoRequest();
        request.setPageSize(100);
        request.setTableGuid(table);
        request.setDataSourceType("odps");
        GetMetaTableFullInfoResponse response = null;
        log.info(JSON.toJSONString(request));
        try {
            response = client.getAcsResponse(request);
            log.info(JSON.toJSONString(response));
            if(response.getSuccess()){
                String tableName=response.getData().getTableName();
                tableFullMap.put(tableName,response.getData().getComment());
                List<GetMetaTableFullInfoResponse.Data.ColumnListItem> cl= response.getData().getColumnList();
                for (GetMetaTableFullInfoResponse.Data.ColumnListItem columnListItem : cl){
                    tableFullMap.put(tableName+columnListItem.getColumnName(),columnListItem.getComment());
                    tableFullMap.put(tableName+columnListItem.getColumnType()+"type",columnListItem.getColumnType());
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
 public static void getDatabase(String pageSize,int pageNum){
     File files = new File("cookie.txt");
     try {
         String cookies=readerMethod(files);
         String url="https://setting.res.cloud.cheryfs.cn/dwproxy/web/di/datasource/list?projectId=10022&tenantId=631873240849152&productCode=di&pageNum="+pageNum+"&searchType=&keyword=&pageSize="+pageSize+"&envType=1";
         String result3= HttpRequest.get(url)
                 .cookie(cookies)
                 .execute().body();
         if(StringUtils.isNotEmpty(result3)){
             JSONObject jsonObject= JSON.parseObject(result3);
             if(jsonObject.getInteger("errCode").toString().equals("0")){
                 JSONObject data=jsonObject.getJSONObject("data");
                 JSONArray list=data.getJSONArray("data");
                 JobVarUtil.databaseTotalCount=data.getInteger("count");
                 for (int i = 0; i < list.size(); i++){
                     JSONObject jsonObject1=list.getJSONObject(i);
                     datasourceMap.put(jsonObject1.getString("displayName"),jsonObject1);
                 }
             }
         }
     } catch (IOException e) {
         throw new RuntimeException(e);
     }
 }

    public static void uploadDataToOdps(String bizDate,List<Record> records,String tableName,String project) throws OdpsException, TunnelException, IOException {
        Account account = new AliyunAccount("feupFUuHYOxrohjn", "MUFrk9HsB94LXzyDZZA0lhPvvoa1J0");
        Odps odps = new Odps(account);
        String odpsUrl = "http://service.cn-wuhu-qrhy-d01.odps.ops.cloud.cheryfs.cn/api";
        odps.setEndpoint(odpsUrl);
        odps.setDefaultProject("datawarehouse_pro");
        TableTunnel tunnel = new TableTunnel(odps);
        Table table = odps.tables().get(tableName);
        PartitionSpec partitionSpec=null;
        if(StringUtils.isNotEmpty(bizDate)){
            partitionSpec=new PartitionSpec();
            partitionSpec.set("ds",bizDate);
            table.deletePartition(partitionSpec,true);
            table.createPartition(partitionSpec);
        }
        TableTunnel.UploadSession uploadSession;
        if (partitionSpec != null) {
            uploadSession = tunnel.createUploadSession(project, tableName, partitionSpec);
        } else {
            uploadSession = tunnel.createUploadSession(project, tableName);
        }
        RecordWriter recordWriter = uploadSession.openRecordWriter(0);
        for (Record record : records) {
            recordWriter.write(record);
        }
        recordWriter.close();
        uploadSession.commit(new Long[]{0L});
    }

    public static void getDataSourceData(String bizDate,String tableName,String project){
        List<Record> records=new ArrayList<>();
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        for (String key:datasourceMap.keySet()){
            JSONObject jsonObject=datasourceMap.get(key);
            Column[] columns = new Column[18];
            columns[0] = new Column("gmtModified", OdpsType.valueOf("STRING"));
            columns[1] = new Column("displayName", OdpsType.valueOf("STRING"));
            columns[2] = new Column("description", OdpsType.valueOf("STRING"));
            columns[3] = new Column("assumeFor", OdpsType.valueOf("STRING"));
            columns[4] = new Column("gmtCreate", OdpsType.valueOf("STRING"));
            columns[5] = new Column("type", OdpsType.valueOf("STRING"));
            columns[6] = new Column("operator", OdpsType.valueOf("STRING"));
            columns[7] = new Column("envType", OdpsType.valueOf("STRING"));
            columns[8] = new Column("name", OdpsType.valueOf("STRING"));
            columns[9] = new Column("tenantId", OdpsType.valueOf("BIGINT"));
            columns[10] = new Column("connection", OdpsType.valueOf("STRING"));
            columns[11] = new Column("subType", OdpsType.valueOf("STRING"));
            columns[12] = new Column("id", OdpsType.valueOf("BIGINT"));
            columns[13] = new Column("connectionShared", OdpsType.valueOf("STRING"));
            columns[14] = new Column("projectId", OdpsType.valueOf("BIGINT"));
            columns[15] = new Column("status", OdpsType.valueOf("STRING"));
            columns[16] = new Column("connIp", OdpsType.valueOf("STRING"));
            columns[17] = new Column("realIp", OdpsType.valueOf("STRING"));
            Record record = new ArrayRecord(columns);
            record.setString(0,sdf.format(jsonObject.getLong("gmtModified")));
            record.setString(1,jsonObject.getString("displayName"));
            record.setString(2,jsonObject.getString("description"));
            record.setString(3,jsonObject.getString("assumeFor"));
            record.setString(4,sdf.format(jsonObject.getLong("gmtCreate")));
            record.setString(5,jsonObject.getString("type").toUpperCase());
            record.setString(6,jsonObject.getString("operator"));
            record.setString(7,jsonObject.getInteger("envType").toString());
            record.setString(8,jsonObject.getString("name"));
            record.setBigint(9,jsonObject.getLong("tenantId"));
            record.setString(10,jsonObject.getString("connection"));
            record.setString(11,jsonObject.getString("subType"));
            record.setBigint(12,jsonObject.getLong("id"));
            record.setString(13,jsonObject.getInteger("connectionShared").toString());
            record.setBigint(14,jsonObject.getLong("projectId"));
            record.setString(15,jsonObject.getInteger("status").toString());
            JSONObject connection=JSON.parseObject(jsonObject.getString("connection"));
            String ip="";
            String type=jsonObject.getString("type");
            if(connection.containsKey("jdbcUrl")){
                String jdbcUrl=connection.getString("jdbcUrl");
                 ip=extractIP(jdbcUrl);
            }else{
                if(type.equals("ftp")){
                    ip=connection.getString("host");
                }
            }
            record.setString(16,ip);
            record.setString(17,extractIP(jsonObject.getString("description")));
            records.add(record);
        }
        try {
            uploadDataToOdps(bizDate,records,tableName,project);
        } catch (OdpsException e) {
            throw new RuntimeException(e);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }
    public static String extractIP(String desc) {
        // 正则表达式匹配IP地址
        String ipPattern = "((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)";
        if(StringUtils.isNotEmpty(desc)){
            Pattern pattern = Pattern.compile(ipPattern);
            Matcher matcher = pattern.matcher(desc);
            if (matcher.find()) {
                return matcher.group();
            }
        }
        return null;
    }



    public static String getLoadBalancers(){
//        try {
//            DescribeLoadBalancersRequest describeLoadBalancersRequest=new DescribeLoadBalancersRequest();
//            describeLoadBalancersRequest.setAcceptFormat(FormatType.JSON);
//            describeLoadBalancersRequest.setPageSize(100);
//            describeLoadBalancersRequest.setPageNumber(1);
//            describeLoadBalancersRequest.setVersion("2014-05-15");
//            describeLoadBalancersRequest.setRegionId("cn-wuhu-qrhy-d01");
//            log.info("请求参数"+JSON.toJSONString(describeLoadBalancersRequest));
//           DescribeLoadBalancersResponse response = slbClient.getAcsResponse(describeLoadBalancersRequest);
//           log.info(JSON.toJSONString(response));
//           return JSON.toJSONString(response);
//        }catch (Exception e){
//            e.printStackTrace();
//        }
        return null;
    }


   public static void checkColumn(List<Map<String,JSONObject>> list){
        Map<String,List<ColumnDto>> res=new HashMap<>();
        List<String> tables=new ArrayList<>();
        List<Record> records=new ArrayList<>();
        try {

            if(datasources.size()==0){
                getDatasource();
            }
            for (Map<String,JSONObject> map:list){
                JSONObject reader=map.get("reader");
                JSONObject writer=map.get("writer");
                JSONObject fileIdJs=map.get("fileId");
                String nodeId=fileIdJs.getString("nodeId");
                String nodeName=fileIdJs.getString("nodeName");
                JSONObject config=map.get("data");
                //    JSONObject prodConfig=new JSONObject(JSON.toJSONString(config));
                String readerType=reader.getString("stepType");
                String writerType=writer.getString("stepType");
                JSONObject js=reader.getJSONObject("parameter");
                JSONObject writerJs=writer.getJSONObject("parameter");
                String readerTableName=js.getString("table");
                String readerDataSource=js.getString("datasource");
                String writerDataSource=writerJs.getString("datasource");
                String writerTableName=writerJs.getString("table");
                List<ColumnDto> readerColum=getAllColumn(readerTableName,readerType,readerDataSource);
                List<ColumnDto> writerColum=getAllColumn(writerTableName,writerType,writerDataSource);
                if(readerColum==null||writerColum==null){
                    continue;
                }
                List<ColumnDto> columnDtos=compareLists(readerColum,writerColum);
                Integer size=0;
                JSONArray array=config.getJSONArray("steps");
                for (int j=0;j<array.size();j++){
                    JSONObject cjs=array.getJSONObject(j);
                    cjs.remove("copies");
                    cjs.remove("gui");
                    if(cjs.getString("name").equals("Reader")){
                        size=cjs.getJSONObject("parameter").getJSONArray("column").size();
                    }
                }
                if(columnDtos.size()>0||readerColum.size()>size){
                    List<String> cls=new ArrayList<>();
                    for (ColumnDto columnDto:columnDtos){
                        cls.add(columnDto.getName().toLowerCase());
                    }
                    String columns= org.apache.commons.lang3.StringUtils.join(cls,",");
                    Column[] colum = new Column[5];
                    colum[0] = new Column("table_name", OdpsType.valueOf("STRING"));
                    colum[1] = new Column("columns_name", OdpsType.valueOf("STRING"));
                    colum[2] = new Column("data_sor_name", OdpsType.valueOf("STRING"));
                    colum[3] = new Column("sys_name", OdpsType.valueOf("STRING"));
                    colum[4] = new Column("node_name", OdpsType.valueOf("STRING"));
                    Record record= new ArrayRecord(colum);
                    if(StringUtils.isNotEmpty(columns)){
                        record.set(0,writerTableName);
                        record.set(1,columns);
                        record.set(2,"");
                        record.set(3,"");
                        if(datasources.containsKey(readerDataSource)){
                            record.set(2,readerDataSource);
                            record.set(3,datasources.get(readerDataSource).getDescription());
                        }
                        record.set(4,nodeName);
                        records.add(record);
                    }
                   }
//                String[] readerColumn=new String[readerColum.size()];
//                String[] writerColumn=new String[readerColum.size()];
//                for(int k=0;k<readerColum.size();k++){
//                    String name=readerColum.get(k).getName();
//                    readerColumn[k]=name;
//                    writerColumn[k]=name.toLowerCase();
//                }
//                JSONArray array=config.getJSONArray("steps");
//                Integer size=0;
//                for (int j=0;j<array.size();j++){
//                    JSONObject cjs=array.getJSONObject(j);
//                    cjs.remove("copies");
//                    cjs.remove("gui");
//                    if(cjs.getString("name").equals("Reader")){
//                        size=cjs.getJSONObject("parameter").getJSONArray("column").size();
//                        cjs.getJSONObject("parameter").put("column",readerColumn);
//                    }
//                    if(cjs.getString("name").equals("Writer")){
//                        cjs.getJSONObject("parameter").put("column",writerColumn);
//                    }
//                    if(cjs.getString("name").equals("Processor")){
//                        config.getJSONArray("steps").remove("Processor");
//                    }
//                }
//                if(columnDtos.size()>0||readerColum.size()>size){
//                    StringBuffer sb=new StringBuffer();
//                    boolean hasPrecision=false;
//                    for (ColumnDto columnDto:writerColum){
//                        String type=columnDto.getType();
//                        if(hasPrecision(type)){
//                            hasPrecision=true;
//                            break;
//                        }
//                    }
//                    if(hasPrecision){
//                        sb.append("set odps.sql.decimal.odps2=true; " );
//                    }
//                    sb.append("alter table "+writerTableName+" add columns ( " );
//                    int i=0;
//                    for (ColumnDto columnDto:columnDtos){
//                        String rtype=columnDto.getType().toUpperCase();
//                        String regex = "\\(.*?\\)";
//                        String rtypes=rtype.replaceAll(regex,"");
//                        String ntype=type.get(rtypes);
//                        if(StringUtils.isEmpty(ntype)){
//                            rtype="STRING";
//                        }else{
//                            rtype=rtype.replace(rtypes,ntype);
//                        }
//                        sb.append(columnDto.getName().toLowerCase()+" "+" "+rtype.toLowerCase()+" comment  "+"'"+columnDto.getComment()+"'" );
//                        i++;
//                        if(i!=columnDtos.size()){
//                            sb.append(" ,");
//                        }
//                    }
//                    sb.append(" );");
//                    tables.add(writerTableName);
//                    excuteSql(sb.toString(),"DataWarehouse_Dev");
//                    excuteSql(sb.toString(),"DataWarehouse_Pro");
//                    log.info(JSON.toJSONString(config));
//                    config.remove("extend");
//                    config.remove("transform");
//                    config.getJSONObject("setting").remove("locale");
//                    config.getJSONObject("setting").put("executeMode",null);
//                    config.getJSONObject("order").getJSONArray("hops").getJSONObject(0).remove("gui");
//                    ListFilesResponse.Data.File file=getFile(nodeId);
//                    if(file!=null){
//                        Long fileId=file.getFileId();
//                        boolean flag=updateDISyncTask(JSON.toJSONString(config),fileId);
//                        if(flag){
//                            submitFile(fileId);
//                        }
//                    }
//                }
            }
            Date newDate = DateUtil.offset(new Date(), DateField.DAY_OF_MONTH, -1);
            String bizDate = DateUtil.format(newDate, "yyyyMMdd");
            if(records.size()>0){
                DataworksUtil.insertInstanceLog(bizDate,records,"dataops_table_columns_diff");
            }
        }catch (Exception e){
            e.printStackTrace();
        }

//       try {
//           Files.write(Paths.get("aaTable.txt"),String.join("|",tables).getBytes(StandardCharsets.UTF_8),StandardOpenOption.TRUNCATE_EXISTING);
//       } catch (IOException e) {
//           e.printStackTrace();
//       }
       log.info(JSON.toJSONString(tables));
   }

   public static List<ColumnDto> getAllColumn(String table,String type,String datasource){
       List<ColumnDto> columnDtos=new ArrayList<>();
        try {
            File files = new File("cookie.txt");
            String cookies=readerMethod(files);
            String url="https://ide.res.cloud.cheryfs.cn/web/di/datasource/table/column?projectId=10022&tenantId=631873240849152&envType=1";
            url+="&stepType="+type+"&datasourceType="+type+"&table="+table+"&datasourceName="+datasource;
            String result= HttpRequest.get(url)
                    .cookie(cookies)
                    .execute().body();
            if(StringUtils.isNotEmpty(result)){
                JSONObject jsonObject=JSON.parseObject(result);
                if(jsonObject.containsKey("errCode")&&jsonObject.getInteger("errCode")==-1){
                    log.info("获取失败:"+table);
                    return  columnDtos;
                }
                JSONArray jsonArray=jsonObject.getJSONObject("data").getJSONArray("columns");
                for (int i=0;i<jsonArray.size();i++){
                    JSONObject jsonObject1=jsonArray.getJSONObject(i);
                    ColumnDto columnDto=new ColumnDto();
                    columnDto.setComment(jsonObject1.getString("comment"));
                    columnDto.setName(jsonObject1.getString("name"));
                    columnDto.setType(jsonObject1.getString("type"));
                    columnDtos.add(columnDto);
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }
        return columnDtos;
   }

    public static List<ColumnDto> compareLists(List<ColumnDto> list1, List<ColumnDto> list2) {
        // 使用 Set 存储 list2 中的 name 字段
        Set<String> namesInList2 = list2.stream()
                .map(ColumnDto::getName)
                .map(String::toLowerCase)
                .collect(Collectors.toSet());

        // 过滤 list1 中不在 list2 中的元素
        return list1.stream()
                .filter(columnDto -> !namesInList2.contains(columnDto.getName().toLowerCase()))
                .collect(Collectors.toList());
    }


    private static  void  excuteSql (String sql,String project){
        Odps odps = new Odps(adminAccount);
        odps.setEndpoint(odpsUrl);
        odps.setDefaultProject(project);
        try {
          Instance i=  SQLTask.run(odps,sql);
          i.waitForSuccess();
        } catch (OdpsException e) {
            throw new RuntimeException(e);
        }
    }


    public  static Table getTableInfo(String tableName,String project){
        Account account = new AliyunAccount("feupFUuHYOxrohjn", "MUFrk9HsB94LXzyDZZA0lhPvvoa1J0");
        Odps odps = new Odps(account);
        String odpsUrl = "http://service.cn-wuhu-qrhy-d01.odps.ops.cloud.cheryfs.cn/api";
        odps.setEndpoint(odpsUrl);
        odps.setDefaultProject(project);
        Table table = odps.tables().get(tableName);//获取当前表
        return table;
    }

    public static boolean hasPrecision(String type) {
        // 使用正则表达式匹配精度信息
        String precisionPattern = "\\((\\d+)(,\\d+)?\\)";
        Pattern pattern = Pattern.compile(precisionPattern);
        Matcher matcher = pattern.matcher(type);
        return matcher.find();
    }

    public static boolean updateDISyncTask(String taskConet,Long fileId){
        boolean flag=false;
        try {
            UpdateDISyncTaskRequest request=new UpdateDISyncTaskRequest();
            request.setProjectId(10022L);
            request.setFileId(fileId);
            request.setTaskContent(taskConet);
            request.setTaskType("DI_OFFLINE");
            log.info(JSON.toJSONString(request));
            UpdateDISyncTaskResponse response = client.getAcsResponse(request);
            if(response.getSuccess()){
                flag=true;
            }
        }catch (Exception e){
            e.printStackTrace();
        }
        return flag;
    }

    public static void deployFile(Long fileId){
        try {
            DeployFileRequest deployFileRequest=new DeployFileRequest();
            deployFileRequest.setFileId(fileId);
            deployFileRequest.setProjectId(10022L);
            deployFileRequest.setComment("字段自动检测补全");
            DeployFileResponse deployFileResponse=client.getAcsResponse(deployFileRequest);
            log.info("字段检测自动补齐发布文件到生产环境:"+JSON.toJSONString(deployFileResponse));
        }catch (Exception e){
            e.printStackTrace();
        }
    }
    public static boolean submitFile(Long fileId){
        boolean flag=false;
        try {
            SubmitFileRequest submitFileRequest=new SubmitFileRequest();
            submitFileRequest.setFileId(fileId);
            submitFileRequest.setProjectId(10022L);
            submitFileRequest.setComment("字段自动检测提交");
            SubmitFileResponse submitFileResponse=client.getAcsResponse(submitFileRequest);
            if(submitFileResponse.getSuccess()){
                flag=true;
                Long deployId=submitFileResponse.getData().longValue();
                getDeployment(deployId,0,fileId);
            }
            log.info("字段检测自动补齐提交文件:"+JSON.toJSONString(submitFileResponse));
        }catch (Exception e){
            e.printStackTrace();
        }
        return flag;
    }
   public static void getDeployment(Long deployId,int count,Long fileId){
       try {
           GetDeploymentRequest getDeploymentRequest=new GetDeploymentRequest();
           getDeploymentRequest.setDeploymentId(deployId);
           getDeploymentRequest.setProjectId(10022L);
           GetDeploymentResponse response=client.getAcsResponse(getDeploymentRequest);
           if(response.getSuccess()){
               Integer status=response.getData().getDeployment().getStatus();
               if(String.valueOf(status).equals("1")){
                   log.info("获取文件发布状态成功:"+JSON.toJSONString(response));
                   deployFile(fileId);
               }else{
                   if(count<4){
                       count+=1;
                       log.info("第"+count+"次获取文件发布状态:"+JSON.toJSONString(response));
                       Thread.sleep(2000);
                       getDeployment(deployId,count,fileId);
                   }
               }
           }
       }catch (Exception e){
           e.printStackTrace();
       }
   }
    public static   ListFilesResponse.Data.File getFile(String nodeId,Long projectId){
        ListFilesRequest listFilesRequest=new ListFilesRequest();
        listFilesRequest.setPageNumber(1);
        listFilesRequest.setPageSize(100);
        listFilesRequest.setProjectId(projectId);
        listFilesRequest.setNodeId(Long.valueOf(nodeId));
        ListFilesResponse.Data.File file=null;
        try {
            ListFilesResponse listFilesResponse=client.getAcsResponse(listFilesRequest);
            if(listFilesResponse.getSuccess()){
                if(listFilesResponse.getData().getFiles().size()>0){
                    return listFilesResponse.getData().getFiles().get(0);
                }
            }
        } catch (ClientException e) {
            e.printStackTrace();
        }
        return file;
    }

    public static JSONObject getFileId(String keyword){
        JSONObject res=new JSONObject();
        File files = new File("cookie.txt");
        try {
            String cookies=readerMethod(files);
            String baseUrl="https://ide.res.cloud.cheryfs.cn/rest/batch/list?projectId=10022&tenantId=631873240849152&batchFilterType=0&useType=0&pageNum=1&pageSize=20&keyword="+keyword;
            String result3= HttpRequest.get(baseUrl)
                    .cookie(cookies)
                    .execute().body();
            if(StringUtils.isNotEmpty(result3)){
                JSONObject jsonObject= JSON.parseObject(result3);
                if(jsonObject.getBoolean("success")){
                     res=jsonObject.getJSONObject("data").getJSONArray("data").getJSONObject(0);
                }
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return res;
    }

    public static JSONObject getFileByFileId(String fileId){
        JSONObject res=new JSONObject();
        File files = new File("cookie.txt");
        try {
            String cookies=readerMethod(files);
            String baseUrl="https://ide.res.cloud.cheryfs.cn/rest/file?projectId=10022&tenantId=631873240849152&openType=0&fileId="+fileId;
            String result3= HttpRequest.get(baseUrl)
                    .cookie(cookies)
                    .execute().body();
            if(StringUtils.isNotEmpty(result3)){
                JSONObject jsonObject= JSON.parseObject(result3);
                if(jsonObject.getBoolean("success")){
                    res=jsonObject.getJSONObject("data");
                }
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return res;
    }
    public static JSONObject putCode(String code){
        JSONObject res=new JSONObject();
        File files = new File("cookie.txt");
        try {
            String cookies=readerMethod(files);
            String baseUrl="https://ide.res.cloud.cheryfs.cn/web/di/code";
            String token=getxcsrfToken();
            String result3= HttpRequest.put(baseUrl)
                    .body(code)
                    .cookie(cookies+" csrf_token="+token+";")
                    .header("x-csrf-token",token)
                    .execute().body();
            if(StringUtils.isNotEmpty(result3)){
                JSONObject jsonObject= JSON.parseObject(result3);
                log.info(JSON.toJSONString(result3));
                if(jsonObject.getBoolean("success")){
                    res=jsonObject.getJSONObject("data");
                }
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return res;
    }
    public static void getDatasource() {
        try {
            ListDataSourcesRequest listDataSourcesRequest = new ListDataSourcesRequest();
            listDataSourcesRequest.setProjectId(10022L);
            int pageNumber = 1;
            int pageSize = 100;
            boolean hasMore = true;
            while (hasMore) {
                listDataSourcesRequest.setPageSize(pageSize);
                listDataSourcesRequest.setPageNumber(pageNumber);
                ListDataSourcesResponse acsResponse = client.getAcsResponse(listDataSourcesRequest);
                List<ListDataSourcesResponse.Data.DataSourcesItem> dataSources = acsResponse.getData().getDataSources();
                if (acsResponse.getData() != null) {
                    dataSources.forEach(datasource -> {
                                String name=datasource.getName();
                                datasources.put(name,datasource);
                            }
                    );

                    if (acsResponse.getData().getDataSources().size() < pageSize) {
                        hasMore = false;
                    } else {
                        pageNumber++;
                    }

                } else {
                    hasMore = false;
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    public static void getDatasource(Long projectId) {
        try {
            ListDataSourcesRequest listDataSourcesRequest = new ListDataSourcesRequest();
            listDataSourcesRequest.setProjectId(projectId);
            int pageNumber = 1;
            int pageSize = 100;
            boolean hasMore = true;
            while (hasMore) {
                listDataSourcesRequest.setPageSize(pageSize);
                listDataSourcesRequest.setPageNumber(pageNumber);
                ListDataSourcesResponse acsResponse = client.getAcsResponse(listDataSourcesRequest);
                List<ListDataSourcesResponse.Data.DataSourcesItem> dataSources = acsResponse.getData().getDataSources();
                if (acsResponse.getData() != null) {
                    dataSources.forEach(datasource -> {
                                String name=datasource.getName();
                                datasources.put(name,datasource);
                            }
                    );

                    if (acsResponse.getData().getDataSources().size() < pageSize) {
                        hasMore = false;
                    } else {
                        pageNumber++;
                    }

                } else {
                    hasMore = false;
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    public static boolean checkFile(String fileName,String bizDate){
        Sftp sftp=null;
        boolean check=false;
        String date=bizDate.replace("-","");
        String year=date.substring(0,4);
        String yearMonth=date.substring(0,6);
        String files= config.getStr(fileName).replace("{DATE}",date).replace("{YARE}",year).replace("{YAERMONTH}",yearMonth);
        String host=config.getStr("ods_file_ip");
        int port=config.getInt("ods_file_port");
        String user=config.getStr("ods_file_user");
        String pwd=config.getStr("ods_file_pwd");
        log.info("任务名:"+fileName);
        log.info("文件名:"+files);
         try{
            sftp= JschUtil.createSftp(host, port, user, pwd);
            boolean ext=sftp.exist(files);
            if(ext){
                check=true;
                SftpATTRS attrs= sftp.getClient().stat(files);
                String time=attrs.getMtimeString();
                Long size=attrs.getSize();
                checkFile.put(fileName,String.valueOf(size));
                log.info("检测文件:"+files+" 文件最后时间:"+time+" 文件大小:"+size);
            }
        }catch (Exception e){
            e.printStackTrace();
        }finally {
          if(sftp!=null){
              sftp.close();
          }
        }
        return check;
    }
    public static GetNodeResponse.Data getNodeByNodeId(Long nodeId){
        GetNodeResponse.Data data=null;
        try {
            GetNodeRequest request=new GetNodeRequest();
            request.setProjectEnv("PROD");
            request.setNodeId(nodeId);
            GetNodeResponse response=client.getAcsResponse(request);
            if(response.getSuccess()){
                data=response.getData();
            }
        }catch (Exception e){
            e.printStackTrace();
        }
       return data;
    }
    public static GetNodeResponse.Data getNodeByNodeId(Long nodeId,Long projectId){
        GetNodeResponse.Data data=null;
        try {
            GetNodeRequest request=new GetNodeRequest();
            request.setProjectEnv("PROD");
            request.setNodeId(nodeId);
            GetNodeResponse response=client.getAcsResponse(request);
            if(response.getSuccess()){
                data=response.getData();
            }
        }catch (Exception e){
            e.printStackTrace();
        }
        return data;
    }

    public static void getMetaTablePartition(){
        try {
            GetMetaTablePartitionRequest request=new GetMetaTablePartitionRequest();
            request.setTableGuid("odps.datawarehouse_pro.ads_f_retl_loan_schedule_busns");
            request.setPageSize(2);
            request.setPageNumber(1);
            GetMetaTablePartitionResponse response=client.getAcsResponse(request);
            if(response.getSuccess()){
                Long total=response.getData().getTotalCount();
                log.info("aa"+total);
                List<GetMetaTablePartitionResponse.Data.DataEntityListItem> data=response.getData().getDataEntityList();
                log.info(JSON.toJSONString(data));
            }
        }catch (Exception e){
            e.printStackTrace();
        }
    }
    public static void importExcel(String tableName,String project,String  ds){
        try {
            // 执行SQL查询
            Account account = new AliyunAccount("feupFUuHYOxrohjn", "MUFrk9HsB94LXzyDZZA0lhPvvoa1J0");
            Odps odps = new Odps(account);
            String odpsUrl = "http://service.cn-wuhu-qrhy-d01.odps.ops.cloud.cheryfs.cn/api";
            odps.setEndpoint(odpsUrl);
            odps.setDefaultProject(project);
            Table table = odps.tables().get(tableName);//获取当前表
            String tabeNameCn=tableName;
            if(StringUtils.isNotEmpty(table.getComment())){
                tabeNameCn=table.getComment();
            }
            TableTunnel tunnel = new TableTunnel(odps);
            PartitionSpec partitionSpec = new PartitionSpec();
            partitionSpec.set("ds",ds);
            List<Record> records=new ArrayList<>();
            try {
                TableTunnel.DownloadSession downloadSession = tunnel.createDownloadSession(project, tableName,
                        partitionSpec);
                log.info("Session Status is : "
                        + downloadSession.getStatus().toString());
                long count = downloadSession.getRecordCount();
                log.info("RecordCount is: " + count);
                RecordReader recordReader = downloadSession.openRecordReader(0,
                        count);
                Record record;
                while ((record = recordReader.read()) != null) {
                    records.add(consumeRecord(record, downloadSession.getSchema()));
                }
                recordReader.close();
            } catch (TunnelException e) {
                e.printStackTrace();
            } catch (IOException e1) {
                e1.printStackTrace();
            }
            // 创建Excel文件
            try (Workbook workbook = new XSSFWorkbook();
                 FileOutputStream fos = new FileOutputStream("./"+tabeNameCn+".xlsx")) {

                Sheet sheet = workbook.createSheet("Sheet1");
                Row headerRow = sheet.createRow(0);

                // 写入表头
                Record record=records.get(0);
                for (int i = 0; i < record.getColumns().length; i++) {
                    Cell cell = headerRow.createCell(i);
                    Column column=record.getColumns()[i];
                    cell.setCellValue(record.getColumns()[i].getName());
                    if(StringUtils.isNotEmpty(column.getComment())){
                        cell.setCellValue(record.getColumns()[i].getComment());
                    }
                }

                // 写入数据
                int rowIndex = 1;
                for (Record rec : records) {
                    Row row = sheet.createRow(rowIndex++);
                    for (int i = 0; i < rec.getColumnCount(); i++) {
                        Cell cell = row.createCell(i);
                        cell.setCellValue("");
                        if(rec.get(i)!=null){
                            cell.setCellValue(rec.get(i).toString());
                        }
                    }
                }

                workbook.write(fos);
                log.info("Excel文件已成功创建。");

            } catch (IOException e) {
                e.printStackTrace();
            }

        } catch (Exception e) {
            String ms= ExceptionUtils.getMessage(e);
            log.info("异常详情:"+ms);
            e.printStackTrace();
        }
    }
    private static Record consumeRecord(Record record, TableSchema schema) {
        for (int i = 0; i < schema.getColumns().size(); i++) {
            Column column = schema.getColumn(i);
            String colValue = null;
            switch (column.getType()) {
                case BIGINT: {
                    Long v = record.getBigint(i);
                    //    colValue = v == null ? null : v.toString();
                    record.set(i,v == null ? null :v);
                    break;
                }
                case BOOLEAN: {
                    Boolean v = record.getBoolean(i);
                    colValue = v == null ? null : v.toString();
                    record.set(i,colValue);
                    break;
                }
                case DATETIME: {
                    Date v = record.getDatetime(i);
                    // colValue = v == null ? null :DateUtil.format(v,"yyyy-MM-dd HH:mm:ss");
                    record.set(i,v == null ? null :v);
                    break;
                }
                case DATE: {
                    Date v = record.getDatetime(i);
                    //colValue = v == null ? null :DateUtil.format(v,"yyyy-MM-dd HH:mm:ss");
                    record.set(i,v == null ? null :v);
                    break;
                }
                case TIMESTAMP: {
                    Date v = record.getDatetime(i);
                    // colValue = v == null ? null :DateUtil.format(v,"yyyy-MM-dd HH:mm:ss");
                    record.set(i,v == null ? null :v);
                    break;
                }
                case DOUBLE: {
                    Double v = record.getDouble(i);
                    colValue = v == null ? null : v.toString();
                    record.set(i,colValue);
                    break;
                }
                case DECIMAL: {
                    String v = record.getString(i);
                    //  colValue = v == null ? null : v.toString();
                    record.set(i,v == null ? null :v);
                    break;
                }
                case STRING: {
                    BigDecimal v = record.getDecimal(i);
                    colValue = v == null ? null : v.toString();
                    record.set(i,colValue);
                    break;
                }
                default:
                    throw new RuntimeException("Unknown column type: "
                            + column.getType());
            }
        }
        return record;
    }

    public static void setSuccessInstance(Long instanceId){
        SetSuccessInstanceRequest request=new SetSuccessInstanceRequest();
        request.setInstanceId(instanceId);
        request.setProjectEnv("PROD");
        try {
            SetSuccessInstanceResponse response=client.getAcsResponse(request);
            if (response.getSuccess()){
                log.info("重置实列:"+instanceId+"成功");
            }
        }catch (Exception e){
            log.info(ExceptionUtils.getMessage(e));
        }
    }

    public static void getFlowName(Long bizId){
        try {
            GetBusinessRequest request=new GetBusinessRequest();
            request.setProjectId(10022L);
            request.setBusinessId(bizId);
            GetBusinessResponse response=client.getAcsResponse(request);
            if(response.getSuccess()){
                log.info(JSON.toJSONString(response.getData()));
            }
        }catch (Exception e){
            e.printStackTrace();
        }
    }
    public static GetFileResponse.Data getFileInfo(Long fileId,Long nodeId){
        try {
            DefaultProfile profile = DefaultProfile.getProfile(
                    "cn-wuhu-qrhy-d01",
                    "rFloKx7rnzOYYaek", // 替换为你的AccessKey ID
                    "rAT0Q9ZD7QKewCAILHIPn92J5a5m1M"); // 替换为你的AccessKey Secret
            DefaultProfile.addEndpoint("cn-wuhu-qrhy-d01", "dataworks-public", "dataworks-public-vpc.res.cloud.cheryfs.cn");
            IAcsClient fileClient = new DefaultAcsClient(profile);
            GetFileRequest request=new GetFileRequest();
            request.setProjectId(10022L);
            request.setFileId(fileId);
            request.setNodeId(nodeId);
            GetFileResponse response=fileClient.getAcsResponse(request);
            if(response.getSuccess()){
                return response.getData();
            }
        }catch (Exception e){
            e.printStackTrace();
        }
        return null;
    }

    public static GetFolderResponse.Data getFolders(String  folderId,Long projectId){
        try {
            GetFolderRequest request=new GetFolderRequest();
            request.setProjectId(projectId);
            request.setFolderId(folderId);
            GetFolderResponse response=client.getAcsResponse(request);
            if(response.getSuccess()){
                return response.getData();
            }
        }catch (Exception e){
            e.printStackTrace();
        }
        return null;
    }

    public static List<GetNodeParentsResponse.Data.NodesItem> getNodeParents(Long nodeId){
        List<GetNodeParentsResponse.Data.NodesItem> list=new ArrayList<>();
        try {
            GetNodeParentsRequest request=new GetNodeParentsRequest();
            request.setNodeId(nodeId);
            request.setProjectEnv("PROD");
            GetNodeParentsResponse response=client.getAcsResponse(request);
            if(response.getSuccess()){
                 list=response.getData().getNodes();
            }
        }catch (Exception e){
            e.printStackTrace();
        }
        return list;
    }

    public static List<ListFileVersionsResponse.Data.FileVersion> getFileVersions(Long project,Long fileId){
        List<ListFileVersionsResponse.Data.FileVersion> list=new ArrayList<>();
        try {
            ListFileVersionsRequest request=new ListFileVersionsRequest();
            request.setFileId(fileId);
            request.setProjectId(project);
            request.setPageSize(100);
            request.setPageNumber(1);
            ListFileVersionsResponse response=client.getAcsResponse(request);
            if(response.getSuccess()){
                list=response.getData().getFileVersions();
            }
        }catch (Exception e){
            e.printStackTrace();
        }
        return list;
    }
    public static String bushu(JSONObject param){
        JSONArray res=new JSONArray();
        String rst="";
        File files = new File("cookie.txt");
        String name="";
        String ext=param.getString("ext");
        String isContainNode=param.getString("containRootNode");
        Date date=new Date();
        try {
            String cookies=readerMethod(files);
            String nodeId=param.getString("nodeId");
            String baseUrl="https://wkbench.res.cloud.cheryfs.cn/workbench/cwf/node/childs?projectId=10022&env=prod&tenantId=631873240849152&nodeId="+nodeId;
            String result3= HttpRequest.get(baseUrl)
                    .cookie(cookies)
                    .execute().body();
            if(StringUtils.isNotEmpty(result3)){
                JSONObject jsonObject= JSON.parseObject(result3);
                JSONObject supplementData=new JSONObject();
                supplementData.put("env","prod");
                supplementData.put("isParallel",false);
                if(jsonObject.getBoolean("success")){
                    res=jsonObject.getJSONArray("data");
                    List<Integer> snode=new ArrayList<>();
                    for (int i=0;i<res.size();i++){
                        JSONObject js=res.getJSONObject(i);
                        String nodeName=js.getString("nodeName");
                        String nodes=js.getString("nodeId");
                        String type=js.getString("prgTypeName");
                        String prgType=js.getString("prgType");
                        if(prgType.equals("6")){
                            continue;
                        }
                        if(nodes.equals(nodeId)){
                            name="P_"+nodeName+"_"+DateUtil.format(date,"yyyyMMdd")+"_"+DateUtil.format(date,"HHmmss");
                        }
                        if(StringUtils.isNotEmpty(isContainNode)&&!isContainNode.equals("true")){
                            if(nodes.equals(nodeId)){
                                continue;
                            }
                        }
                        if(StringUtils.isEmpty(ext)){
                            snode.add(Integer.valueOf(nodes));
                        }else{
                            if(!nodeName.toLowerCase().contains(ext.toLowerCase())){
                                snode.add(Integer.valueOf(nodes));
                            }
                        }
                    }
                    supplementData.put("name",name);
                    supplementData.put("includeNodeIds",snode);
                    supplementData.put("order","asc");
                    supplementData.put("parallelGroup","1");
                    supplementData.put("projectId",10022);
                    supplementData.put("rootNodeId",Integer.valueOf(nodeId));
                    supplementData.put("rootNodeProjectId",1022);
                    supplementData.put("tenantId",631873240849152L);
                    supplementData.put("useMultipleTimePeriods",true);
                    JSONArray biz=new JSONArray();
                    JSONObject bizTime=new JSONObject();
                    bizTime.put("bizBeginTime",param.getString("bizStart"));
                    bizTime.put("bizEndTime",param.getString("bizEnd"));
                    biz.set(0,bizTime);
                    supplementData.put("multipleTimePeriods",JSON.toJSONString(biz));
                    String token=getcsrfToken();
                    HttpRequest request= HttpRequest.post("https://wkbench.res.cloud.cheryfs.cn/workbench/cwf/node/supplementdata?projectId=10022&env=prod&tenantId=631873240849152");
                    String result= request.header(Header.CONTENT_TYPE, "application/json")
                            .header(Header.ACCEPT, "application/json")
                            .header(Header.COOKIE, cookies+"; "+"csrf-token="+token)
                            .header("x-csrf-token",token)
                            .body(supplementData.toJSONString())
                            .execute().body();
                    if(StringUtils.isNotEmpty(result)){
                        log.info(JSON.toJSONString(result));
                    }
                }else{
                    rst=jsonObject.getString("errMsg");
                }
            }
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return rst;
    }
    public static  Map<String,ListProjectMembersResponse.Data.ProjectMember> getMembers(Long projectId){
        Map<String,ListProjectMembersResponse.Data.ProjectMember> map=new ConcurrentHashMap<>();
        try {
            ListProjectMembersRequest request=new ListProjectMembersRequest();
            request.setProjectId(projectId);
            int pageNumber = 1;
            int pageSize = 20;
            boolean hasMore = true;
            while (hasMore) {
                request.setPageSize(pageSize);
                request.setPageNumber(pageNumber);
                ListProjectMembersResponse acsResponse = client.getAcsResponse(request);
                if (acsResponse.getData() != null) {
                    acsResponse.getData().getProjectMemberList().forEach(member -> {
                                String id=member.getProjectMemberId();
                                map.put(id,member);
                            }
                    );
                    if (acsResponse.getData().getProjectMemberList().size() < pageSize) {
                        hasMore = false;
                    } else {
                        pageNumber++;
                    }

                } else {
                    hasMore = false;
                }
            }
        }catch (Exception e){
            e.printStackTrace();
        }
        return map;
    }
    public static GetMetaTableBasicInfoResponse.Data  getMetaTableBasicInfo(String tableGuid){
        GetMetaTableBasicInfoResponse.Data data=new GetMetaTableBasicInfoResponse.Data();
        try {
            GetMetaTableBasicInfoRequest request=new GetMetaTableBasicInfoRequest();
            request.setTableGuid(tableGuid);
            request.setExtension(true);
            GetMetaTableBasicInfoResponse response=client.getAcsResponse(request);
            if(response.getSuccess()){
                data=response.getData();
            }
        }catch (Exception e){
            e.printStackTrace();
        }
        return data;
    }

     public static String exactTableName(String sql){
        String tableName=null;
         // 移除所有注释（以--开头直到行尾的内容）
         String noCommentSql = sql.replaceAll("--.*", "");

         // 定义正则表达式（不区分大小写）
         Pattern pattern = Pattern.compile(
                 "insert\\s+(?:into|overwrite)\\s+(?:table\\s+)?(\\S+)\\s+partition",
                 Pattern.CASE_INSENSITIVE
         );

         Matcher matcher = pattern.matcher(noCommentSql);
         if (matcher.find()) {
             tableName=matcher.group(1); // 直接返回表名
         }
         if(StringUtils.isEmpty(tableName)){
             pattern = Pattern.compile(
                     "insert\\s+(?:into|overwrite)\\s+([^\\s(]+)\\s*(?:\\(|\\s+select\\s)",
                     Pattern.CASE_INSENSITIVE|Pattern.DOTALL
             );
             matcher = pattern.matcher(noCommentSql);
             if (matcher.find()) {
                 tableName=matcher.group(1); // 直接返回表名
             }
         }
        return tableName;
     }

    public static void main(String[] args) {
       // String log=getNodeByNodeId(19550L);
       // getLoadBalancers();
       // String log= "";
       // System.out.println(LogParserUtil.getlogErrorInfo(log));
      //  getFile("37643");
      //  getFolders("sqjoncoo9tysggf5nxoq7kgq");
       // getNodeParents(40908L);
       getDataSourceInfoByNodeCode("21825");
//        Long fileId= getFile("17910",10022L).getFileId();
//        Map<String,ListProjectMembersResponse.Data.ProjectMember> map=getMembers(10022L);
//        List<ListFileVersionsResponse.Data.FileVersion> versions=getFileVersions(10022L,fileId);
//        if(versions!=null&&versions.size()>0){
//            String memberId=versions.get(0).getCommitUser();
//            System.out.println(JSON.toJSONString(map.get(memberId)));
//        }
   //     getMetaTableBasicInfo("odps.datawarehouse_pro.dws_t01_adm_area");
//        DataTaskInfoDto aa= getDataSourceInfoByNodeCode("20907");
//        System.out.println(JSON.toJSONString(aa));
        // 35853 36037 18847 22982
    //    getMetaTableBasicInfo("odps.datawarehouse_pro."+exactTableName(getNodeCode("22982")));

       // DataworksUtil.phoneWarning("下周值班请保持手机畅通","18838098890","15617319916","15617319916");
        //folderDi
       // getFolder(15912L);
     //   getMetaTablePartition();
       // System.out.println(log.length());
    }






}
